Data summary and statistical analysis notebook
Subeida Hassan
26th August 2022
- Preliminary analysis
- Joining the two datasets
- Research Question 1
- 1. What affect does the assessment type have on the student performance and the BAME awarding gap?
- Statistical Analysis/Modelling
- Research Question 2
- 2. What affect does the assessment type have on the performance of Non-White students (Namely Asian, Black, Chinese, and mixed) and the awarding gap?
- Statistical Analysis
- Research Question 3
- 3. Is there a link between assessment type, gender and student achievement?
- Statistical Analysis
Loading required packages
#importing libraries
library("tidyverse")
library(here)
library(ggrepel)
library(lme4)
library(lmerTest)
library(sjstats)
library(emmeans)
library(MuMIn)
library(sjPlot)
library(sjmisc)
library("car")Preliminary analysis
Reading data into data frame
#Ethnicity data set
ethnicity_data <- read_delim(file = here("data", "qry_Ethnic_origin_2.txt"), delim = ",")Rows: 44191 Columns: 3
── Column specification ───────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (2): GENDER_DESC, ETHNIC_ORIGIN_DESC
dbl (1): ID2
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.head(ethnicity_data)
#Mark data set
mark_data <- read_delim(file = here("data", "qry_Mark_Data_2.txt"), delim = ",")Rows: 1117342 Columns: 22
── Column specification ───────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (13): PROGRAMME_SCHOOL, PROGRAMME_OWNER, PROGRAMME_LEVEL_DESC, PROGRAMME_PART, PART_END...
dbl (9): ID2, STUDENT_START_YEAR, PART_START_ACADEMIC_YEAR, INSTANCE_ACADEMIC_YEAR, MODULE...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.head(mark_data)Ethnicity Dataset
Cleaning
#checking for missing data
ethnicity_data %>% sapply(function(x) sum(is.na(x)) ) ID2 GENDER_DESC ETHNIC_ORIGIN_DESC
0 1 15 #removing missing data
clean_ethnicity_data <- ethnicity_data %>%
drop_na()
#checking if na's removed.
clean_ethnicity_data %>% sapply(function(x) sum(is.na(x))) ID2 GENDER_DESC ETHNIC_ORIGIN_DESC
0 0 0 #checking for duplicates & removing
length(unique(clean_ethnicity_data$ID2)) #number of students in ethnic data[1] 44176length(unique(clean_ethnicity_data$ID2)) == nrow(clean_ethnicity_data) #returns TRUE as no duplicates[1] TRUE#renaming columns
clean_ethnicity_data %>%
rename(
Gender = GENDER_DESC,
Ethnicity = ETHNIC_ORIGIN_DESC
) -> clean_ethnicity_data#count of ethnic groups / adding column for % of each ethnicity
#check levels of gender
clean_ethnicity_data %>%
group_by(Gender) %>%
tally () %>%
mutate(freq = (n / sum(n))*100) %>%
mutate(percentage = scales::label_percent()(n / sum(n))) %>%
rename("Number of students" = n)
#Categorising into man vs woman other levels to little cases
#remove non binary, other & prefer not to say
clean_ethnicity_data<-clean_ethnicity_data[!(clean_ethnicity_data$Gender=="Non-Binary" | clean_ethnicity_data$Gender=="Prefer not to say" | clean_ethnicity_data$Gender=="Other") ,]
#check levels of ethnicites
clean_ethnicity_data %>%
group_by(Ethnicity) %>%
tally () %>%
mutate(freq = (n / sum(n))*100) %>%
mutate(percentage = scales::label_percent()(n / sum(n)))I will categorise ethnicity differently depending on research question. Firstly, I will begin with white vs other.
#duplicating data set at this point as will categorise differently for second research question
clean_ethnicity_data -> clean_ethnicity_data2#categorise ethnicities into white vs non white
#rename all white columns- white.
#rename all other columns other.
clean_ethnicity_data[clean_ethnicity_data == "White - British"] <- "White"
clean_ethnicity_data[clean_ethnicity_data == "White - Irish"] <- "White"
clean_ethnicity_data[clean_ethnicity_data == "White - Scottish"] <- "White"
clean_ethnicity_data[clean_ethnicity_data == "Other White Background"] <- "White"
clean_ethnicity_data[clean_ethnicity_data == "Arab"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Asian Or Asian British - Bangladeshi"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Asian Or Asian British - Indian"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Asian Or Asian British - Pakistani"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Black Or Black British - African"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Black Or Black British - Caribbean"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Chinese"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Gypsy or Traveller"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Mixed - White And Asian"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Mixed - White And Black African"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Mixed - White And Black Caribbean"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Other Asian Background"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Other Black Background"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Other Ethnic Background"] <- "BAME"
clean_ethnicity_data[clean_ethnicity_data == "Other Mixed Background"] <- "BAME"
#remove not known,no information & info refused- as could be any ethnicity.
clean_ethnicity_data<-clean_ethnicity_data[!(clean_ethnicity_data$Ethnicity=="Information Refused" | clean_ethnicity_data$Ethnicity=="Information Refused - OBSOLETE VALUE" | clean_ethnicity_data$Ethnicity=="No Information" | clean_ethnicity_data$Ethnicity=="Not Known") ,]
#changing numerical variables to factors.
clean_ethnicity_data %>%
mutate_at(vars(Gender, Ethnicity), list(factor)) -> clean_ethnicity_data
summary(clean_ethnicity_data) ID2 Gender Ethnicity
Min. : 1 Man :24727 BAME :17389
1st Qu.:11045 Woman:18025 White:25363
Median :22124
Mean :22101
3rd Qu.:33137
Max. :44191 Visualisation
#Ethnicity
#Visualising number of students in each ethnicity category
ethnicity_data %>%
group_by(ETHNIC_ORIGIN_DESC) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))*100) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(., aes(x = ETHNIC_ORIGIN_DESC, y = counts)) +
geom_bar(fill = "#0073C2FF", stat = "identity") +
#geom_text(aes(label = percentage), angle = 60) +
theme(axis.text.x = element_text(angle = 60,
hjust = 1)) +
ggtitle("Number of students in each ethnic category") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Ethnicity') +
ylab('Number of students') 
#Ethnicity
#Percentage of students belonging to each ethnicity after categorisation
clean_ethnicity_data %>%
group_by(Ethnicity) %>%
tally () %>%
mutate(freq = scales::label_percent()(n / sum(n))) %>%
ggplot( ., aes(x="", y=freq, fill=Ethnicity)) +
geom_bar(stat="identity", width=1, color="white") +
coord_polar("y", start=0) +
theme_void() +
geom_text(aes(label = freq), color = "white", position = position_stack(vjust = 0.5), size = 4) +
scale_fill_brewer(palette = "Paired") +
ggtitle("Percentage of students in each ethnic category") +
theme(plot.title = element_text(hjust = 0.5)) 
#GENDER
#Percentage of students belonging to each gender category
ethnicity_data %>%
group_by(GENDER_DESC) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))*100) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(., aes(x = GENDER_DESC, y = counts, fill=GENDER_DESC)) +
geom_bar(stat = "identity") +
geom_text(aes(label = percentage), vjust = -0.5) +
scale_fill_brewer(palette = "Paired") +
ggtitle("Percentage of students in each gender category") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Gender') +
ylab('Number of students') +
guides(fill = guide_legend(title = "Gender Description ")) 
#GENDER
#Percentage of students belonging to each gender after categorisation
clean_ethnicity_data %>%
group_by(Gender) %>%
tally () %>%
mutate(freq = scales::label_percent()(n / sum(n))) %>%
ggplot( ., aes(x="", y=freq, fill=Gender)) +
geom_bar(stat="identity", width=1, color="white") +
coord_polar("y", start=0) +
theme_void() +
geom_text(aes(label = freq), color = "white", position = position_stack(vjust = 0.5), size = 5) +
scale_fill_brewer(palette = "Set2") +
ggtitle("Percentage of students in each gender category") +
theme(plot.title = element_text(hjust = 0.5)) 
Mark Dataset
Cleaning
#summary of mark data
summary(mark_data) ID2 STUDENT_START_YEAR PROGRAMME_SCHOOL PROGRAMME_OWNER PROGRAMME_LEVEL_DESC
Min. : 1 Min. : 7.0 Length:1117342 Length:1117342 Length:1117342
1st Qu.:11013 1st Qu.:17.0 Class :character Class :character Class :character
Median :22121 Median :18.0 Mode :character Mode :character Mode :character
Mean :22094 Mean :17.8
3rd Qu.:33118 3rd Qu.:19.0
Max. :44191 Max. :22.0
PROGRAMME_PART PART_START_ACADEMIC_YEAR PART_END_DATE MODULE_OWNER
Length:1117342 Min. : 8.00 Length:1117342 Length:1117342
Class :character 1st Qu.:18.00 Class :character Class :character
Mode :character Median :19.00 Mode :character Mode :character
Mean :18.66
3rd Qu.:20.00
Max. :21.00
MODULE_SCHOOL MODULE_CODE MODULE_LONG_TITLE INSTANCE_ACADEMIC_YEAR
Length:1117342 Length:1117342 Length:1117342 Min. :17.0
Class :character Class :character Class :character 1st Qu.:18.0
Mode :character Mode :character Mode :character Median :19.0
Mean :18.7
3rd Qu.:20.0
Max. :21.0
MODULE_INSTANCE_NO MODULE_MARK INCLUDE_MARK_DESC MODULE_CREDITS
Min. : 1.000 Min. : 0.00 Length:1117342 Min. : 10.0
1st Qu.: 1.000 1st Qu.: 58.00 Class :character 1st Qu.: 10.0
Median : 1.000 Median : 65.00 Mode :character Median : 15.0
Mean : 1.217 Mean : 63.94 Mean : 16.4
3rd Qu.: 1.000 3rd Qu.: 72.00 3rd Qu.: 20.0
Max. :59.000 Max. :100.00 Max. :120.0
NA's :8
ASSESSMENT_COMPONENT_CODE ASSESSMENT_COMPONENT_TITLE ASSESSMENT_COMPONENT_TYPE_DESC
Length:1117342 Length:1117342 Length:1117342
Class :character Class :character Class :character
Mode :character Mode :character Mode :character
ASSESSMENT_WEIGHT MARK
Min. : 0.00 Min. : -3.00
1st Qu.: 20.00 1st Qu.: 58.00
Median : 40.00 Median : 66.00
Mean : 48.34 Mean : 66.18
3rd Qu.: 75.00 3rd Qu.: 75.00
Max. :100.00 Max. :100.00
#checking for missing data?
mark_data %>% sapply(function(x) sum(is.na(x)) ) ID2 STUDENT_START_YEAR PROGRAMME_SCHOOL
0 0 20509
PROGRAMME_OWNER PROGRAMME_LEVEL_DESC PROGRAMME_PART
0 0 0
PART_START_ACADEMIC_YEAR PART_END_DATE MODULE_OWNER
0 130170 0
MODULE_SCHOOL MODULE_CODE MODULE_LONG_TITLE
20509 0 0
INSTANCE_ACADEMIC_YEAR MODULE_INSTANCE_NO MODULE_MARK
0 0 8
INCLUDE_MARK_DESC MODULE_CREDITS ASSESSMENT_COMPONENT_CODE
0 0 0
ASSESSMENT_COMPONENT_TITLE ASSESSMENT_COMPONENT_TYPE_DESC ASSESSMENT_WEIGHT
0 0 0
MARK
0 #remove missing data??? - decided to not remove missing data as only missing data in programme school, module school and part_end_date and module mark- i am not directly assessing these columns- so doesn't matter too much. If looking at effect of school on mark may need to consider removing missing values.
#checking for duplicates
length(unique(mark_data$ID2)) #number of students in marks data- same as ethnicity data set[1] 44191length(unique(mark_data$ID2)) == nrow(mark_data) #returns FALSE as there are duplicates[1] FALSE#removing duplicate rows- 1,114,911 rows once duplicates removed.
distinct(mark_data) -> clean_mark_data#changing variables to factors.
clean_mark_data %>%
mutate_at(vars(PROGRAMME_SCHOOL, PROGRAMME_OWNER, PROGRAMME_LEVEL_DESC, PROGRAMME_PART, PART_END_DATE, MODULE_OWNER, MODULE_SCHOOL, MODULE_CODE, MODULE_LONG_TITLE, INCLUDE_MARK_DESC, ASSESSMENT_COMPONENT_CODE, ASSESSMENT_COMPONENT_TITLE, ASSESSMENT_COMPONENT_TYPE_DESC), list(factor)) -> clean_mark_dataVisualisation
#how many years of data?
clean_mark_data %>%
group_by(STUDENT_START_YEAR) %>%
count ()#data spans 2007-2022
#Only interested in the last 5 years, as before 2016, a lot less data/outdated.
#mark data only has a single students results for 2022, therefore, only one gender, one ethnicity.
#This is not good enough for any analysis on students performance, will cause serious over generalisation.
#remove data before 2017 & after 2021
clean_mark_data %>%
filter(STUDENT_START_YEAR >= 17 & STUDENT_START_YEAR <= 21) ->clean_mark_data
#Visualisation of students in each year group.
clean_mark_data %>%
group_by(STUDENT_START_YEAR) %>%
count() %>%
ggplot(., aes(x = STUDENT_START_YEAR, y= n)) +
geom_bar(stat = "identity", fill = "darkgreen", size = 2.5) +
geom_text(aes(label = n), vjust = -0.5) +
ggtitle("Number of students in each year (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Student Start Year') +
ylab('Number of students') 
#now only 886,263 rows#How many assessment types in data set?
clean_mark_data %>%
group_by(ASSESSMENT_COMPONENT_TYPE_DESC) %>%
count()#Coursework, Exam, In-Class Test, In-Dept Exam, Laboratory Test
#types of assessment types & relative percentages of students who did the AT
clean_mark_data %>%
group_by(ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot( ., aes(x="", y=freq, fill=ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_bar(stat="identity", width=1, color="white") +
coord_polar("y", start=180) +
theme_void() +
geom_text(aes(label = percentage), position = position_stack(vjust = 0.5), size = 3) +
scale_fill_brewer(palette = "Spectral") +
guides(fill = guide_legend(title = "Assessment type")) +
ggtitle("Percentage of students in each assessment type") +
theme(plot.title = element_text(hjust = 0.5)) 
#fix labels?? or change to bar chart
#change to dark2?
#categorise into exam and cw only.
#So- exam = Exam, In-Dept Exam, cw = CW, In-Class Test and Lab Test
clean_mark_data[clean_mark_data == "In-Dept Exam"] <- "Exam"Warning: stack imbalance in '==', 41 then 43
Warning: stack imbalance in '==', 25 then 27
Warning: stack imbalance in '[<-', 10 then 12
Warning: stack imbalance in '<-', 2 then 4clean_mark_data[clean_mark_data == "In-Class Test"] <- "Coursework"
clean_mark_data[clean_mark_data == "Laboratory Test"] <- "Coursework"
#dropping levels
#in dept exam, in class test and lab test
droplevels(clean_mark_data$ASSESSMENT_COMPONENT_TYPE_DESC) -> clean_mark_data$ASSESSMENT_COMPONENT_TYPE_DESC
summary(clean_mark_data$ASSESSMENT_COMPONENT_TYPE_DESC)Coursework Exam
618851 267412 #visualising percentage of students under each assessment type after categorisation
clean_mark_data %>%
group_by(ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot( ., aes(x="", y=freq, fill=ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_bar(stat="identity", width=1, color="white") +
coord_polar("y", start=0) +
theme_void() +
geom_text(aes(label = percentage), position = position_stack(vjust = 0.5), size = 3) +
scale_fill_brewer(palette = "Spectral") +
guides(fill = guide_legend(title = "Assessment type")) +
ggtitle("Percentage of students in each assessment type") +
theme(plot.title = element_text(hjust = 0.5)) 
#cleaning programme part
summary(clean_mark_data$PROGRAMME_PART) A B C D F T
343429 229228 97993 6310 18717 174344 ggplot(clean_mark_data, aes(x = PROGRAMME_PART)) +
geom_bar(fill = "blue") +
theme(axis.text.x = element_text(angle = 60,
hjust = 1)) +
ggtitle("Number of students in each part") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Programme Part') +
ylab('Number of students') 
As can be seen from the above table- there are many parts- but in this study i am only interested in UG and PG students, so will remove all cases where not A, B, C, D, F, and T.
#as can be seen from the above table- there are many parts- but in this study i am only interested in UG and PG students, so will remove all cases where not A, B, C, D, F, and T.
#There are 16 parts- want to keep 6 and remove 10
subset(clean_mark_data, PROGRAMME_PART != "G" & PROGRAMME_PART != "I" & PROGRAMME_PART != "R" & PROGRAMME_PART != "R0" & PROGRAMME_PART != "R1" & PROGRAMME_PART != "R2" & PROGRAMME_PART != "RT" & PROGRAMME_PART != "TO" & PROGRAMME_PART != "TR" & PROGRAMME_PART != "TX") -> clean_mark_data
#dropping levels with 0
summary(clean_mark_data$PROGRAMME_PART) A B C D F G I R R0 R1 R2 RT T
343430 229228 97993 6310 18717 0 0 0 0 0 0 0 174344
TO TR TX
0 0 0 droplevels(clean_mark_data$PROGRAMME_PART) -> clean_mark_data$PROGRAMME_PART
summary(clean_mark_data$PROGRAMME_PART) A B C D F T
343430 229228 97993 6310 18717 174344 ggplot(clean_mark_data, aes(x = PROGRAMME_PART)) +
geom_bar(fill = "blue") +
theme(axis.text.x = element_text(angle = 60,
hjust = 1)) +
ggtitle("Number of students in each part") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Programme Part') +
ylab('Number of students')
#distribution of marks in cleaned mark dataset
hist(clean_mark_data$MARK,
main="Histogram of mark (2017-2021)",
xlab="Mark",
xlim=c(0,100),
col="darkmagenta")
#seems close to normal distribution#from above looks like there are marks below 0
clean_mark_data %>%
group_by(MARK) %>%
tally ()#REMOVE MARKS BELOW, ONE ENTRY OF -3.
filter(clean_mark_data, MARK > 0) -> clean_mark_data
#now 870021 rowsJoining the two datasets
#JOIN ETHNIC DATA AND MARK DATA
inner_join(clean_ethnicity_data, clean_mark_data) -> full_dataJoining, by = "ID2"print(distinct(full_data))#840368 rowsResearch Question 1
1. What affect does the assessment type have on the student performance and the BAME awarding gap?
This is a multi-leveled question. I will began by looking at the effect of ethnicity and assessment type separately on student performance.
1.1 Effect of Ethnicity on student performance
Variance in marks for white vs other
boxplot(MARK~Ethnicity, data = full_data)
looks like a lot of data points are outliers- but data is over 5 years and we are mostly interested in top grades so will filter these out later.
Analysis of average Marks of White vs BAME/Other students (2017-2021)
Below is bar chart showing of effect of ethnicity on mean mark between 2017-2021
full_data %>%
select(Ethnicity, MARK) %>%
group_by(Ethnicity) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
ggplot(aes(x=Ethnicity, y = Average_mark, fill =Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Paired") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Test Scores by Ethnicity between 2017-2021") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Average Mark')
Next, I wanted to see the affect of ethnicity on average mark each year between 2017-2021. This table below shows the gap in average grade in white vs Bame students is reducing over the years. This reduction is very small changing by just 1.4 over 5 years.
#White vs non white grades table like in screenshot in report
full_data %>%
select(INSTANCE_ACADEMIC_YEAR, Ethnicity,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, Ethnicity) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
spread(key = Ethnicity, value = Average_mark) %>%
mutate(Gap = White - BAME)Below is a line graph to visualise the above table. This is showing gap is reducing, but both white and bame students average grade seems to be reducing over the years.
full_data %>%
select(INSTANCE_ACADEMIC_YEAR, Ethnicity,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, Ethnicity) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
# mutate(Gap = White - BAME) %>%
ggplot(aes(x = INSTANCE_ACADEMIC_YEAR, y = Average_mark, col = Ethnicity)) +
geom_line() +
geom_point() +
ggtitle("Average grade for white vs BAME students (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Average Mark') +
labs(col='Ethnicity') 
Analysis of BAME Awarding Gap (2017-2021)
The below bar graph is showing percentage of white vs BAME students achieving top degree (2017-2021) NOTE- for methodology/results I grouped by ID2 first in order to calculate average grade for each student..then i filtered for students who got an average grade above 60 Then grouped by ethnicity This is important as the data is not important, if not grouped by ID2 results would show any student who got above 60 for at least 1 assessment
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Ethnicity, Average_mark) %>%
distinct() %>%
group_by(Ethnicity) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Ethnicity, y = counts, fill =Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position = position_stack(vjust = 1.1), size = 3) +
ggtitle("Percentage of white vs BAME students achieving top degree (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Number of students')
Below graph shows difference between number of White vs BAME students achieving top degree (BAME awarding gap) is consistent over years 2017-2021
#Visualising percentage of white vs BAME students achieving top degree each year (2017-2021)
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Ethnicity, Average_mark, INSTANCE_ACADEMIC_YEAR) %>%
distinct() %>%
group_by(INSTANCE_ACADEMIC_YEAR, Ethnicity) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=INSTANCE_ACADEMIC_YEAR, y = counts, fill =Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Percentage of white vs BAME students achieving top degree each year (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Number of students')`summarise()` has grouped output by 'INSTANCE_ACADEMIC_YEAR'. You can override using the `.groups` argument.
#this shows awarding gap is reducing but only by 4% between 2017-2020, 2021 less data so not good for comparisonProblem with above graphs is that there are a lot more white students in the university, and therefore it is a given that more white students are achieving top degrees. So instead, this graph clearly shows distribution of degree outcomes instead.The following graph makes the results clearer for interpretation/comparisons. For example here, 37% of white students get a first in comparison to only 21.9% of BAME students.
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
mutate(Grade = case_when(Average_mark >= 70 ~ "First Class",
Average_mark >= 60 ~ "Upper Second Class Honour",
Average_mark >= 50 ~ "Lower Second Class Honour",
Average_mark >= 40 ~ "Third class/pass",
Average_mark < 40 ~ "Fail")) %>%
group_by(Ethnicity, Grade) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Ethnicity, y = freq, fill =Grade)) +
geom_bar(stat = 'identity') +
coord_flip() +
scale_fill_brewer(palette = "Pastel1") +
geom_text_repel(aes(label = percentage), position = position_stack(vjust = 0.5), size = 2.5) +
ggtitle("Distribution of degree outcomes by ethnic group (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
labs(y = "Frequency", fill = "Degree Classification") #+`summarise()` has grouped output by 'Ethnicity'. You can override using the `.groups` argument.
#scale_fill_discrete(breaks=c('First Class', 'Upper Second Class Honour', 'Lower Second Class Honour', 'Third class/pass', 'Fail'))1.2 Effect of assessment type on student performance
I have began by visualising the average mark for each assessment type (2017-2021)
#Visualise effect of assessment type on average mark between 2017-2021
full_data %>%
select(ASSESSMENT_COMPONENT_TYPE_DESC, MARK) %>%
group_by(ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = Average_mark, fill = ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel2") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Mark for Coursework vs Exam (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Average Mark') +
guides(fill = guide_legend(title = "Assessment type")) 
This bar graph above is showing that students overall perform much better in coursework than exams…but does ethnicity have an effect on this?? Will look at this later
Next i wanted to evaluate the average yearly mark dependent on assessment type. This shows every year the average grade is much bigger for cw type assessments vs exams. Essentially students perform much better in coursework type assessments in comparison to exams.
full_data %>%
select(INSTANCE_ACADEMIC_YEAR, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
spread(key = ASSESSMENT_COMPONENT_TYPE_DESC, value = Average_mark) %>%
mutate(Gap = Coursework - Exam)This line graph below shows many things: 1. students perform much better in cw vs exams 2. Average cw mark is reducing over the years 3. Average exam mark is increasing over the years, but goes down in 2021 4. assessment awarding gap is reducing.
#Visualisation of above
full_data %>%
select(INSTANCE_ACADEMIC_YEAR, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = ASSESSMENT_COMPONENT_TYPE_DESC, value = Average_mark) %>%
#mutate(Gap = Coursework - Exam) %>%
ggplot(aes(x = INSTANCE_ACADEMIC_YEAR, y = Average_mark, col = ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_line() +
geom_point() +
ggtitle("Average yearly Mark for Coursework vs Exam (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Average Mark') +
labs(col='Assessment Type')
Analysis of effect of assessment type on achieving top degree(2017-2022)
What about effect of assessment type on top degrees? This bar chart shows the average number of students that did cw vs exam in all students that achieved a top degree.
#average percentage of students who did cw vs exam to get first
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, ASSESSMENT_COMPONENT_TYPE_DESC, Average_mark) %>%
distinct() %>%
group_by(ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = counts, fill = ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel2") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Percentage of students doing cw vs exam for top degree") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Number of students') +
guides(fill = guide_legend(title = "Assessment type")) 
This bar chart is showing yearly average percentage of students who did cw vs exam to get first This results are pretty consistent over the years. Has reduced slightly over the years. Gap in 21 looks pretty small, but number of students in this year was lower.
#percentage of students who did cw vs exam to get first (2017-2022)
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, ASSESSMENT_COMPONENT_TYPE_DESC, Average_mark, INSTANCE_ACADEMIC_YEAR) %>%
distinct() %>%
group_by(INSTANCE_ACADEMIC_YEAR, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=INSTANCE_ACADEMIC_YEAR, y = counts, fill =ASSESSMENT_COMPONENT_TYPE_DESC)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Percentage of students doing cw vs exam for top degree each year") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Number of students') +
guides(fill = guide_legend(title = "Assessment type")) `summarise()` has grouped output by 'INSTANCE_ACADEMIC_YEAR'. You can override using the `.groups` argument.
1.3 Analysing effect of assessment type on student performance and BAME awarding gap
#Checking there is a reasonable number of white/others in each assessment type.
full_data %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
group_by(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally()First i want to look at affect of assessment type on student performance(White vs Other) This table below shows the difference in avgerage cw and exam marks for white Vs Bame students
Average grade over past 5 years for assessment type (AT) and ethnicity
full_data %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
spread(key = Ethnicity, value = Average_mark) %>%
mutate(gap = White - BAME)The gap doesnt look huge, but it is bigger for coursework type assessments. However this is gap in average mark (student performance), not the number of students who achieved a top degree. Therefore, this is not relating to Bame awarding gap. Still interesting…
Visualisation of above table Looking at average grade over past 5 years for AT and ethnicity
full_data %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = Average_mark, fill = Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Accent") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Mark By Assessment Type And Ethnicity") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') 
ylab('Average Mark')$y
[1] "Average Mark"
attr(,"class")
[1] "labels"This graph above is useful as it shows difference in average mark for each assessment type by ethnicity. However this research question pertains to the affect of assessment type on BAME awarding gaps. Therefore, instead of average marks, we need to compare the number of white vs BAME students who achieved top degrees and see if gap is greater for cw type assessments or coursework.
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC, Average_mark) %>%
distinct() %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
group_by(Ethnicity,ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
#select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC, percentage) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = percentage, fill = Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Set1") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Effect of assessment type on awarding gap") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Percentage of students')Adding missing grouping variables: `ID2`
`summarise()` has grouped output by 'Ethnicity'. You can override using the `.groups` argument.
The results here are interesting. There is a 4% awarding gap for both ethnicities in each assessment type- this suggests exam do not lead to a wider awarding gap for the years 2017-2021.White students still performing better for both assessment types, but both ethnicities performing similar for assessment type.
#interaction plot of full dataset.
interaction.plot(x.factor = full_data$Ethnicity, #x-axis variable
trace.factor = full_data$ASSESSMENT_COMPONENT_TYPE_DESC, #variable for lines
response = full_data$MARK, #y-axis variable
fun = median, #metric to plot
ylab = "Average Mark",
xlab = "Ethnicity",
col = c("red", "blue"),
lty = 1, #line type
lwd = 2, #line width
trace.label = "Assessment type")
Statistical Analysis/Modelling
Mixed model 1
#Question 1:
# 1. What affect does the assessment type have on student performance?
#fitting model
mmodel1 <- lmer(MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity + PROGRAMME_PART + (1|ID2) + (1|MODULE_CODE), data = full_data)summary(mmodel1)Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity + PROGRAMME_PART +
(1 | ID2) + (1 | MODULE_CODE)
Data: full_data
REML criterion at convergence: 6688052
Scaled residuals:
Min 1Q Median 3Q Max
-7.4683 -0.5331 0.0220 0.5787 4.7933
Random effects:
Groups Name Variance Std.Dev.
ID2 (Intercept) 38.83 6.232
MODULE_CODE (Intercept) 25.44 5.043
Residual 153.51 12.390
Number of obs: 840368, groups: ID2, 31540; MODULE_CODE, 6945
Fixed effects:
Estimate Std. Error df t value
(Intercept) 6.537e+01 1.521e-01 1.530e+04 429.825
ASSESSMENT_COMPONENT_TYPE_DESCExam -8.123e+00 5.911e-02 7.942e+05 -137.425
EthnicityWhite 3.333e+00 9.279e-02 3.443e+04 35.920
PROGRAMME_PARTB 5.245e-02 1.613e-01 1.550e+04 0.325
PROGRAMME_PARTC 7.652e-01 1.897e-01 1.295e+04 4.033
PROGRAMME_PARTD -1.309e+00 3.776e-01 3.151e+04 -3.467
PROGRAMME_PARTF 5.816e+00 5.340e-01 6.400e+03 10.892
PROGRAMME_PARTT -3.757e-01 1.903e-01 1.176e+04 -1.974
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite 3.466e-01 6.729e-02 8.341e+05 5.151
Pr(>|t|)
(Intercept) < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam < 2e-16 ***
EthnicityWhite < 2e-16 ***
PROGRAMME_PARTB 0.744988
PROGRAMME_PARTC 5.53e-05 ***
PROGRAMME_PARTD 0.000527 ***
PROGRAMME_PARTF < 2e-16 ***
PROGRAMME_PARTT 0.048427 *
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite 2.59e-07 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation of Fixed Effects:
(Intr) ASSESSMENT_COMPONENT_TYPE_DESCEx EthncW
ASSESSMENT_COMPONENT_TYPE_DESCEx -0.130
EthnictyWht -0.414 0.155
PROGRAMME_PARTB -0.548 -0.001 0.003
PROGRAMME_PARTC -0.525 0.000 0.006
PROGRAMME_PARTD -0.281 0.004 0.011
PROGRAMME_PARTF -0.216 0.003 0.008
PROGRAMME_PARTT -0.716 0.037 0.155
ASSESSMENT_COMPONENT_TYPE_DESCE: 0.083 -0.709 -0.214
PROGRAMME_PARTB PROGRAMME_PARTC PROGRAMME_PARTD
ASSESSMENT_COMPONENT_TYPE_DESCEx
EthnictyWht
PROGRAMME_PARTB
PROGRAMME_PARTC 0.510
PROGRAMME_PARTD 0.245 0.328
PROGRAMME_PARTF 0.153 0.145 0.075
PROGRAMME_PARTT 0.440 0.427 0.255
ASSESSMENT_COMPONENT_TYPE_DESCE: -0.003 0.000 0.000
PROGRAMME_PARTF PROGRAMME_PARTT
ASSESSMENT_COMPONENT_TYPE_DESCEx
EthnictyWht
PROGRAMME_PARTB
PROGRAMME_PARTC
PROGRAMME_PARTD
PROGRAMME_PARTF
PROGRAMME_PARTT 0.170
ASSESSMENT_COMPONENT_TYPE_DESCE: -0.001 -0.010
optimizer (nloptwrap) convergence code: 0 (OK)
Model failed to converge with max|grad| = 0.00454631 (tol = 0.002, component 1)#interaction plot of model...
plot_model(mmodel1, type = "pred", terms = c("Ethnicity", "ASSESSMENT_COMPONENT_TYPE_DESC"))
anova(mmodel1)Type III Analysis of Variance Table with Satterthwaite's method
Sum Sq Mean Sq NumDF DenDF F value Pr(>F)
ASSESSMENT_COMPONENT_TYPE_DESC 5366486 5366486 1 717508 34959.266 < 2.2e-16 ***
Ethnicity 224560 224560 1 32669 1462.867 < 2.2e-16 ***
PROGRAMME_PART 27274 5455 5 12989 35.534 < 2.2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESC:Ethnicity 4073 4073 1 834072 26.532 2.593e-07 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1#shows significance of all these variables on mark??Results from the anova test show each of these variables have significant effect on mark
#effect size...look at video for how to explain this in report.
effectsize::eta_squared(mmodel1, partial = TRUE)# Effect Size for ANOVA (Type III)
Parameter | Eta2 (partial) | 95% CI
------------------------------------------------------------------------
ASSESSMENT_COMPONENT_TYPE_DESC | 0.05 | [0.05, 1.00]
Ethnicity | 0.04 | [0.04, 1.00]
PROGRAMME_PART | 0.01 | [0.01, 1.00]
ASSESSMENT_COMPONENT_TYPE_DESC:Ethnicity | 3.18e-05 | [0.00, 1.00]
- One-sided CIs: upper bound fixed at [1.00].#emmeans is meant to be looked at for main effect.
#in this case they were all significant so will do for each??
emmeans(mmodel1, list(pairwise~ASSESSMENT_COMPONENT_TYPE_DESC), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of ASSESSMENT_COMPONENT_TYPE_DESC`
ASSESSMENT_COMPONENT_TYPE_DESC emmean SE df asymp.LCL asymp.UCL
Coursework 67.86 0.1263 Inf 67.62 68.11
Exam 59.91 0.1290 Inf 59.66 60.17
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of ASSESSMENT_COMPONENT_TYPE_DESC`
1 estimate SE df z.ratio p.value
Coursework - Exam 7.95 0.0425 Inf 186.974 <.0001
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
Degrees-of-freedom method: asymptotic #here comparison shows that there is a sig diff in mark when assessment type is cw compared to exam (p.value = <0.0001)VarCorr(mmodel1) Groups Name Std.Dev.
ID2 (Intercept) 6.2315
MODULE_CODE (Intercept) 5.0433
Residual 12.3898 totSD <- sqrt( 6.2315 + 5.0433 + 12.3898)
emmV <- emmeans(mmodel1, ~ ASSESSMENT_COMPONENT_TYPE_DESC)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactionseff_size(emmV, sigma = totSD, edf = 5) contrast effect.size SE df asymp.LCL asymp.UCL
Coursework - Exam 1.63 0.517 Inf 0.621 2.65
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
sigma used for effect sizes: 4.865
Degrees-of-freedom method: inherited from asymptotic when re-gridding
Confidence level used: 0.95 emmeans(mmodel1, list(pairwise~Ethnicity), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of Ethnicity`
Ethnicity emmean SE df asymp.LCL asymp.UCL
BAME 62.14 0.1373 Inf 61.87 62.41
White 65.64 0.1305 Inf 65.39 65.90
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of Ethnicity`
1 estimate SE df z.ratio p.value
BAME - White -3.51 0.0917 Inf -38.247 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic #same interpretation as above but for ethnicity??? but ask safa why is other-white not white-other.emmV2 <- emmeans(mmodel1, ~ Ethnicity)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactionseff_size(emmV2, sigma = totSD, edf = 5) contrast effect.size SE df asymp.LCL asymp.UCL
BAME - White -0.721 0.229 Inf -1.17 -0.273
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
sigma used for effect sizes: 4.865
Degrees-of-freedom method: inherited from asymptotic when re-gridding
Confidence level used: 0.95 emmeans(mmodel1, list(pairwise~PROGRAMME_PART), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.$`emmeans of PROGRAMME_PART`
PROGRAMME_PART emmean SE df asymp.LCL asymp.UCL
A 63.06 0.1387 Inf 62.79 63.34
B 63.12 0.1356 Inf 62.85 63.38
C 63.83 0.1582 Inf 63.52 64.14
D 61.76 0.3606 Inf 61.05 62.46
F 68.88 0.5194 Inf 67.86 69.90
T 62.69 0.1308 Inf 62.43 62.95
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Ethnicity
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of PROGRAMME_PART`
1 estimate SE df z.ratio p.value
A - B -0.0525 0.161 Inf -0.325 0.9995
A - C -0.7652 0.190 Inf -4.033 0.0008
A - D 1.3093 0.378 Inf 3.467 0.0070
A - F -5.8159 0.534 Inf -10.892 <.0001
A - T 0.3757 0.190 Inf 1.974 0.3574
B - C -0.7127 0.175 Inf -4.062 0.0007
B - D 1.3617 0.372 Inf 3.656 0.0035
B - F -5.7634 0.534 Inf -10.800 <.0001
B - T 0.4281 0.188 Inf 2.282 0.2014
C - D 2.0744 0.363 Inf 5.717 <.0001
C - F -5.0507 0.540 Inf -9.350 <.0001
C - T 1.1408 0.203 Inf 5.609 <.0001
D - F -7.1252 0.630 Inf -11.304 <.0001
D - T -0.9336 0.377 Inf -2.476 0.1312
F - T 6.1916 0.536 Inf 11.561 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Ethnicity
Degrees-of-freedom method: asymptotic
P value adjustment: tukey method for comparing a family of 6 estimates #interesting as only some parts show sig difference on mark... some parts e.g. B,T, show no sig diff on mark?? wonder why?? is this correct interpretation??emmeans(mmodel1, pairwise ~ Ethnicity | ASSESSMENT_COMPONENT_TYPE_DESC)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.$emmeans
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
Ethnicity emmean SE df asymp.LCL asymp.UCL
BAME 66.20 0.1380 Inf 65.93 66.47
White 69.53 0.1311 Inf 69.27 69.79
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
Ethnicity emmean SE df asymp.LCL asymp.UCL
BAME 58.07 0.1428 Inf 57.79 58.35
White 61.75 0.1345 Inf 61.49 62.02
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$contrasts
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
contrast estimate SE df z.ratio p.value
BAME - White -3.33 0.0928 Inf -35.920 <.0001
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
contrast estimate SE df z.ratio p.value
BAME - White -3.68 0.1023 Inf -35.974 <.0001
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic emmV3 <- emmeans(mmodel1, ~ Ethnicity | ASSESSMENT_COMPONENT_TYPE_DESC)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.eff_size(emmV3, sigma = totSD, edf = 5)ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
contrast effect.size SE df asymp.LCL asymp.UCL
BAME - White -0.685 0.218 Inf -1.11 -0.259
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
contrast effect.size SE df asymp.LCL asymp.UCL
BAME - White -0.756 0.240 Inf -1.23 -0.286
Results are averaged over the levels of: PROGRAMME_PART
sigma used for effect sizes: 4.865
Degrees-of-freedom method: inherited from asymptotic when re-gridding
Confidence level used: 0.95 Evaluating mixed model-1 assumptions
linearity
Usually, there are 3 model assumptions which we need to check; linearity, homoscedasticity and normality of residuals. We do not need to look at linearity as all variables in this model are categorical.
Homoscedasticity:
#code adapted from https://ademos.people.uic.edu/Chapter18.html
plot(mmodel1)
boxplot(mmodel1@resp$wtres~full_data$ASSESSMENT_COMPONENT_TYPE_DESC)
boxplot(mmodel1@resp$wtres~full_data$Ethnicity)
boxplot(mmodel1@resp$wtres~full_data$PROGRAMME_PART)
Another way to check for homoscedasticity - (sd) residuals
#ASSESSMENT TYPE
mmodel1res<-mmodel1@resp$wtres
#assessment type residuals in df
mmodel1df<-data.frame(full_data$ASSESSMENT_COMPONENT_TYPE_DESC,mmodel1res)
names(mmodel1df)<-c("ASSESSMENT_COMPONENT_TYPE_DESC","residual")
mmodel1df %>% group_by(ASSESSMENT_COMPONENT_TYPE_DESC)%>% summarise(sd(residual)) Since the highest sd is less than twice the lowest, that is fine (according to general rule of thumb for homoscedasticity across groups of a factor)
Now we will check for the remaining variables…
#Ethnicity
mmodel1df2<-data.frame(full_data$Ethnicity,mmodel1res)
names(mmodel1df2)<-c("Ethnicity","residual")
mmodel1df2 %>% group_by(Ethnicity)%>% summarise(sd(residual))
#assumption satisfied #programme part
mmodel1df3<-data.frame(full_data$PROGRAMME_PART,mmodel1res)
names(mmodel1df3)<-c("Programme_Part","residual")
mmodel1df3 %>% group_by(Programme_Part)%>% summarise(sd(residual))
#assumption met for this categorical variable as the largest residual is 15.3 and is less than 2* the smallest residual of D (9.20)normality of residuals
qqnorm(mmodel1res)
Confidence intervals
#Interval estimates for mixed models
#confint(mmodel1)The above model is assessing effect of variables on student performance by looking at average marks. We want to look at awarding gaps so will categorise the marks into top degree and below and run a binomial model.
Generalised linear mixed model 1.2
#creating binomial response variable
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
mutate(Grade = case_when(Average_mark >= 60 ~ 1,
Average_mark < 60 ~ 0)) -> full_data3
#1. What affect does the assessment type have on the BAME awarding gap?
linmod<-glmer(as.factor(Grade) ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity + PROGRAMME_PART + (1|ID2) + (1|MODULE_CODE), family="binomial",data = full_data3, nAGQ=0)summary(linmod)Generalized linear mixed model fit by maximum likelihood (Adaptive Gauss-Hermite Quadrature,
nAGQ = 0) [glmerMod]
Family: binomial ( logit )
Formula: as.factor(Grade) ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity +
PROGRAMME_PART + (1 | ID2) + (1 | MODULE_CODE)
Data: full_data3
AIC BIC logLik deviance df.resid
52332.8 52460.9 -26155.4 52310.8 840357
Scaled residuals:
Min 1Q Median 3Q Max
-0.18149 0.01053 0.01402 0.01861 0.07701
Random effects:
Groups Name Variance Std.Dev.
ID2 (Intercept) 3.070e+02 1.752e+01
MODULE_CODE (Intercept) 6.814e-12 2.610e-06
Number of obs: 840368, groups: ID2, 31540; MODULE_CODE, 6945
Fixed effects:
Estimate Std. Error z value Pr(>|z|)
(Intercept) 3.5760 0.2312 15.466 < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam -0.1197 0.1662 -0.720 0.47152
EthnicityWhite 2.3745 0.2582 9.196 < 2e-16 ***
PROGRAMME_PARTB 0.4248 0.1625 2.614 0.00895 **
PROGRAMME_PARTC 0.5394 0.2511 2.148 0.03170 *
PROGRAMME_PARTD 1.9214 1.8806 1.022 0.30694
PROGRAMME_PARTF -0.5267 0.5435 -0.969 0.33257
PROGRAMME_PARTT -0.1385 0.2593 -0.534 0.59341
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite -0.0483 0.2357 -0.205 0.83768
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation of Fixed Effects:
(Intr) ASSESSMENT_COMPONENT_TYPE_DESCEx EthncW
ASSESSMENT_COMPONENT_TYPE_DESCEx -0.237
EthnictyWht -0.728 0.196
PROGRAMME_PARTB -0.185 -0.019 0.006
PROGRAMME_PARTC -0.134 -0.009 0.007
PROGRAMME_PARTD -0.020 0.003 0.002
PROGRAMME_PARTF -0.131 0.008 0.030
PROGRAMME_PARTT -0.687 0.095 0.392
ASSESSMENT_COMPONENT_TYPE_DESCE: 0.144 -0.701 -0.272
PROGRAMME_PARTB PROGRAMME_PARTC PROGRAMME_PARTD
ASSESSMENT_COMPONENT_TYPE_DESCEx
EthnictyWht
PROGRAMME_PARTB
PROGRAMME_PARTC 0.282
PROGRAMME_PARTD 0.037 0.038
PROGRAMME_PARTF 0.080 0.052 0.008
PROGRAMME_PARTT 0.168 0.122 0.017
ASSESSMENT_COMPONENT_TYPE_DESCE: -0.001 0.000 -0.002
PROGRAMME_PARTF PROGRAMME_PARTT
ASSESSMENT_COMPONENT_TYPE_DESCEx
EthnictyWht
PROGRAMME_PARTB
PROGRAMME_PARTC
PROGRAMME_PARTD
PROGRAMME_PARTF
PROGRAMME_PARTT 0.107
ASSESSMENT_COMPONENT_TYPE_DESCE: 0.001 -0.030
optimizer (bobyqa) convergence code: 0 (OK)
boundary (singular) fit: see help('isSingular')se <- sqrt(diag(vcov(linmod)))
# table of estimates with 95% CI
(tab <- cbind(Est = fixef(linmod), LL = fixef(linmod) - 1.96 * se, UL = fixef(linmod) + 1.96 *
se)) Est LL UL
(Intercept) 3.57595111 3.12276002 4.0291422
ASSESSMENT_COMPONENT_TYPE_DESCExam -0.11965227 -0.44536946 0.2060649
EthnicityWhite 2.37450940 1.86844174 2.8805771
PROGRAMME_PARTB 0.42478261 0.10625209 0.7433131
PROGRAMME_PARTC 0.53936330 0.04726069 1.0314659
PROGRAMME_PARTD 1.92137632 -1.76467657 5.6074292
PROGRAMME_PARTF -0.52666637 -1.59200991 0.5386772
PROGRAMME_PARTT -0.13846290 -0.64677264 0.3698468
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite -0.04829525 -0.51035739 0.4137669exp(tab) Est LL UL
(Intercept) 35.7285865 22.7089705 56.212671
ASSESSMENT_COMPONENT_TYPE_DESCExam 0.8872289 0.6405876 1.228833
EthnicityWhite 10.7457400 6.4781938 17.824556
PROGRAMME_PARTB 1.5292579 1.1121022 2.102891
PROGRAMME_PARTC 1.7149146 1.0483953 2.805175
PROGRAMME_PARTD 6.8303527 0.1712422 272.442940
PROGRAMME_PARTF 0.5905704 0.2035162 1.713738
PROGRAMME_PARTT 0.8706956 0.5237333 1.447513
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite 0.9528524 0.6002810 1.512504#evaluating model significance/quality
#Analysis of Deviance
#install.packages("car")
#library("car")
Anova(linmod)Analysis of Deviance Table (Type II Wald chisquare tests)
Response: as.factor(Grade)
Chisq Df Pr(>Chisq)
ASSESSMENT_COMPONENT_TYPE_DESC 1.4648 1 0.22616
Ethnicity 90.2155 1 < 2e-16 ***
PROGRAMME_PART 12.3015 5 0.03088 *
ASSESSMENT_COMPONENT_TYPE_DESC:Ethnicity 0.0420 1 0.83768
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1plot_model(linmod, type = "pred", terms = c("Ethnicity", "ASSESSMENT_COMPONENT_TYPE_DESC"))
emmeans(linmod, ~ ASSESSMENT_COMPONENT_TYPE_DESC, type = "response")NOTE: Results may be misleading due to involvement in interactions ASSESSMENT_COMPONENT_TYPE_DESC response SE df asymp.LCL asymp.UCL
Coursework 0.9941 0.002031 Inf 0.9885 0.9970
Exam 0.9932 0.002389 Inf 0.9865 0.9966
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
Unknown transformation "as.factor": no transformation done
Confidence level used: 0.95 emmeans(linmod, ~ Ethnicity, type = "response")NOTE: Results may be misleading due to involvement in interactions Ethnicity response SE df asymp.LCL asymp.UCL
BAME 0.9799 0.0074121 Inf 0.9589 0.9903
White 0.9980 0.0007064 Inf 0.9960 0.9990
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Unknown transformation "as.factor": no transformation done
Confidence level used: 0.95 emmeans(linmod, ~ Ethnicity | ASSESSMENT_COMPONENT_TYPE_DESC, type = "response")ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
Ethnicity response SE df asymp.LCL asymp.UCL
BAME 0.9810 0.0070326 Inf 0.9610 0.9909
White 0.9982 0.0006544 Inf 0.9963 0.9991
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
Ethnicity response SE df asymp.LCL asymp.UCL
BAME 0.9787 0.0081866 Inf 0.9551 0.9900
White 0.9979 0.0008030 Inf 0.9955 0.9990
Results are averaged over the levels of: PROGRAMME_PART
Unknown transformation "as.factor": no transformation done
Confidence level used: 0.95 This model does not require checking of assumptions
Research Question 2
2. What affect does the assessment type have on the performance of Non-White students (Namely Asian, Black, Chinese, and mixed) and the awarding gap?
Data wrangling: This research question required different categorisisation of ethnicities.
#categorising ethnicities into Asian, Black, Chinese, Mixed & White
clean_ethnicity_data2[clean_ethnicity_data2 == "White - British"] <- "White"
clean_ethnicity_data2[clean_ethnicity_data2 == "White - Irish"] <- "White"
clean_ethnicity_data2[clean_ethnicity_data2 == "White - Scottish"] <- "White"
clean_ethnicity_data2[clean_ethnicity_data2 == "Other White Background"] <- "White"
clean_ethnicity_data2[clean_ethnicity_data2 == "Asian Or Asian British - Bangladeshi"] <- "Asian"
clean_ethnicity_data2[clean_ethnicity_data2 == "Asian Or Asian British - Indian"] <- "Asian"
clean_ethnicity_data2[clean_ethnicity_data2 == "Asian Or Asian British - Pakistani"] <- "Asian"
clean_ethnicity_data2[clean_ethnicity_data2 == "Black Or Black British - African"] <- "Black"
clean_ethnicity_data2[clean_ethnicity_data2 == "Black Or Black British - Caribbean"] <- "Black"
clean_ethnicity_data2[clean_ethnicity_data2 == "Chinese"] <- "Chinese"
clean_ethnicity_data2[clean_ethnicity_data2 == "Mixed - White And Asian"] <- "Mixed"
clean_ethnicity_data2[clean_ethnicity_data2 == "Mixed - White And Black African"] <- "Mixed"
clean_ethnicity_data2[clean_ethnicity_data2 == "Mixed - White And Black Caribbean"] <- "Mixed"
clean_ethnicity_data2[clean_ethnicity_data2 == "Other Asian Background"] <- "Asian"
clean_ethnicity_data2[clean_ethnicity_data2 == "Other Black Background"] <- "Black"
clean_ethnicity_data2[clean_ethnicity_data2 == "Other Mixed Background"] <- "Mixed"
#remove not known,no information & info refused- as could be any ethnicity.
#also removed gypsy as only 1 student- not good for representation
#also removed other ethnic background- as not sure what category this relates to
clean_ethnicity_data2<-clean_ethnicity_data2[!(clean_ethnicity_data2$Ethnicity=="Information Refused" | clean_ethnicity_data2$Ethnicity=="Information Refused - OBSOLETE VALUE" | clean_ethnicity_data2$Ethnicity=="No Information" | clean_ethnicity_data2$Ethnicity=="Not Known" | clean_ethnicity_data2$Ethnicity=="Gypsy or Traveller" | clean_ethnicity_data2$Ethnicity=="Other Ethnic Background" | clean_ethnicity_data2$Ethnicity=="Arab") ,]clean_ethnicity_data2 %>%
mutate_at(vars(Gender, Ethnicity), list(factor)) -> clean_ethnicity_data2
summary(clean_ethnicity_data2) ID2 Gender Ethnicity
Min. : 1 Man :24318 Asian : 5408
1st Qu.:11056 Woman:17820 Black : 2814
Median :22126 Chinese: 6688
Mean :22104 Mixed : 1865
3rd Qu.:33135 White :25363
Max. :44191 #Ethnicity
#Percentage of students belonging to each ethnicity after categorisation
clean_ethnicity_data2 %>%
group_by(Ethnicity) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot( ., aes(x="", y=freq, fill=Ethnicity)) +
geom_bar(stat="identity", width=1, color="white") +
coord_polar("y", start=0) +
theme_void() +
geom_text(aes(label = percentage), color = "white", position = position_stack(vjust = 0.5), size = 2.5) +
scale_fill_brewer(palette = "Paired") +
ggtitle("Percentage of students in each ethnic category") +
theme(plot.title = element_text(hjust = 0.5)) 
Joining two datasets
#JOIN ETHNIC DATA AND MARK DATA
inner_join(clean_ethnicity_data2, clean_mark_data) -> full_data2
print(distinct(full_data2))2.1 Effect of Ethncicity (Non-White students) on student performance
boxplot(MARK~Ethnicity, data = full_data2)
#Visualise effect of ethnicity on overall/mean mark between 2017-2021
full_data2 %>%
select(Ethnicity, MARK) %>%
group_by(Ethnicity) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
ggplot(aes(x=Ethnicity, y = Average_mark, fill =Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Paired") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Test Scores by Ethnicity between 2017-2021") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Average Mark') 
#looking at avg grade for each ethnicity type between 2017-2021
full_data2 %>%
select(INSTANCE_ACADEMIC_YEAR, Ethnicity,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, Ethnicity) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
ggplot(aes(x = INSTANCE_ACADEMIC_YEAR, y = Average_mark, col = Ethnicity)) +
geom_line() +
geom_point() +
ggtitle("Average grade for White vs BAME (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Average Mark') +
labs(col='Ethnicity') 
Analysis of Non-white students Awarding Gap (2017-2021)
#Visualising percentage of white vs BAME students achieving top degree (2017-2021)
full_data2 %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Ethnicity, Average_mark) %>%
distinct() %>%
group_by(Ethnicity) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Ethnicity, y = counts, fill =Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position = position_stack(vjust = 1.1), size = 3) +
ggtitle("Percentage of white vs BAME students achieving top degree (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Number of students')
#BAME AWARDING GAP
full_data2 %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
mutate(Grade = case_when(Average_mark >= 70 ~ "First Class",
Average_mark >= 60 ~ "Upper Second Class Honour",
Average_mark >= 50 ~ "Lower Second Class Honour",
Average_mark >= 40 ~ "Third class/pass",
Average_mark < 40 ~ "Fail")) %>%
group_by(Ethnicity, Grade) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Ethnicity, y = freq, fill =Grade)) +
geom_bar(stat = 'identity') +
coord_flip() +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position = position_stack(vjust = 0.5), size = 2.5) +
ggtitle("Distribution of degree outcomes by ethnic group (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
labs(y = "Frequency", fill = "Degree Classification")`summarise()` has grouped output by 'Ethnicity'. You can override using the `.groups` argument.
2.2 Analysing effect of assessment type on non white students performance and awarding gap
full_data2 %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = Average_mark, fill = Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Set1") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Mark By Assessment Type And Ethnicity") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Average Mark')
Effect of assessment type on awarding gap for Non-white categorise.
full_data2 %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC, Average_mark) %>%
distinct() %>%
select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
group_by(Ethnicity,ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
#select(Ethnicity, ASSESSMENT_COMPONENT_TYPE_DESC, percentage) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = percentage, fill = Ethnicity)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Set1") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Effect of assessment type on awarding gap") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Percentage of students')Adding missing grouping variables: `ID2`
`summarise()` has grouped output by 'Ethnicity'. You can override using the `.groups` argument.
interaction.plot(x.factor = full_data2$Ethnicity, #x-axis variable
trace.factor = full_data2$ASSESSMENT_COMPONENT_TYPE_DESC, #variable for lines
response = full_data2$MARK, #y-axis variable
fun = median, #metric to plot
ylab = "Mark",
xlab = "Ethnicity",
col = c("pink", "lightblue"),
lty = 1, #line type
lwd = 2, #line width
trace.label = "Assessment type")
Statistical Analysis
Mixed Model 2
#Question 2:
# 2. What affect does the assessment type have on the performance of Non-White students (Namely Asian, Black, Chinese, and mixed)
mmodel2<- lmer(MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity + PROGRAMME_PART + (1|ID2) + (1|MODULE_CODE), data = full_data2)
summary(mmodel2)Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Ethnicity + PROGRAMME_PART +
(1 | ID2) + (1 | MODULE_CODE)
Data: full_data2
REML criterion at convergence: 6585014
Scaled residuals:
Min 1Q Median 3Q Max
-7.4776 -0.5326 0.0220 0.5783 4.8132
Random effects:
Groups Name Variance Std.Dev.
ID2 (Intercept) 37.93 6.158
MODULE_CODE (Intercept) 25.28 5.028
Residual 153.15 12.375
Number of obs: 827711, groups: ID2, 31062; MODULE_CODE, 6939
Fixed effects:
Estimate Std. Error df t value
(Intercept) 6.634e+01 1.747e-01 2.325e+04 379.791
ASSESSMENT_COMPONENT_TYPE_DESCExam -8.116e+00 8.779e-02 8.194e+05 -92.448
EthnicityBlack -1.485e+00 1.895e-01 3.176e+04 -7.835
EthnicityChinese -3.433e+00 1.631e-01 3.652e+04 -21.052
EthnicityMixed 9.747e-01 2.231e-01 3.176e+04 4.369
EthnicityWhite 2.288e+00 1.269e-01 3.332e+04 18.035
PROGRAMME_PARTB 4.924e-02 1.615e-01 1.522e+04 0.305
PROGRAMME_PARTC 7.788e-01 1.898e-01 1.277e+04 4.102
PROGRAMME_PARTD -1.202e+00 3.782e-01 3.099e+04 -3.179
PROGRAMME_PARTF 5.688e+00 5.341e-01 6.438e+03 10.649
PROGRAMME_PARTT 3.115e-01 1.935e-01 1.250e+04 1.609
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityBlack -1.281e+00 1.393e-01 8.183e+05 -9.197
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityChinese 1.423e+00 1.449e-01 8.248e+05 9.823
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityMixed -1.760e-01 1.620e-01 8.175e+05 -1.086
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite 3.375e-01 9.320e-02 8.203e+05 3.622
Pr(>|t|)
(Intercept) < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam < 2e-16 ***
EthnicityBlack 4.84e-15 ***
EthnicityChinese < 2e-16 ***
EthnicityMixed 1.25e-05 ***
EthnicityWhite < 2e-16 ***
PROGRAMME_PARTB 0.760414
PROGRAMME_PARTC 4.12e-05 ***
PROGRAMME_PARTD 0.001478 **
PROGRAMME_PARTF < 2e-16 ***
PROGRAMME_PARTT 0.107560
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityBlack < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityChinese < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityMixed 0.277308
ASSESSMENT_COMPONENT_TYPE_DESCExam:EthnicityWhite 0.000293 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation matrix not shown by default, as p = 15 > 12.
Use print(x, correlation=TRUE) or
vcov(x) if you need it#same as above- but shows more values ...plot_model(mmodel2, type = "pred", terms = c("Ethnicity", "ASSESSMENT_COMPONENT_TYPE_DESC"))
anova(mmodel2)Type III Analysis of Variance Table with Satterthwaite's method
Sum Sq Mean Sq NumDF DenDF F value Pr(>F)
ASSESSMENT_COMPONENT_TYPE_DESC 3439838 3439838 1 764529 22460.340 < 2.2e-16 ***
Ethnicity 277515 69379 4 32481 453.007 < 2.2e-16 ***
PROGRAMME_PART 23368 4674 5 13145 30.516 < 2.2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESC:Ethnicity 46918 11730 4 822301 76.588 < 2.2e-16 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1Similar to RQ1, all variables hold a significant effect on student performance
#effect size...look at video for how to explain this in report.
effectsize::eta_squared(mmodel2, partial = TRUE)# Effect Size for ANOVA (Type III)
Parameter | Eta2 (partial) | 95% CI
------------------------------------------------------------------------
ASSESSMENT_COMPONENT_TYPE_DESC | 0.03 | [0.03, 1.00]
Ethnicity | 0.05 | [0.05, 1.00]
PROGRAMME_PART | 0.01 | [0.01, 1.00]
ASSESSMENT_COMPONENT_TYPE_DESC:Ethnicity | 3.72e-04 | [0.00, 1.00]
- One-sided CIs: upper bound fixed at [1.00].#emmeans is meant to be looked at for main effect.
#in this case they were all significant so will do for each??
emmeans(mmodel2, list(pairwise~ASSESSMENT_COMPONENT_TYPE_DESC), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 827711' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 827711' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of ASSESSMENT_COMPONENT_TYPE_DESC`
ASSESSMENT_COMPONENT_TYPE_DESC emmean SE df asymp.LCL asymp.UCL
Coursework 66.94 0.1335 Inf 66.68 67.20
Exam 58.89 0.1376 Inf 58.62 59.16
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of ASSESSMENT_COMPONENT_TYPE_DESC`
1 estimate SE df z.ratio p.value
Coursework - Exam 8.06 0.0538 Inf 149.868 <.0001
Results are averaged over the levels of: Ethnicity, PROGRAMME_PART
Degrees-of-freedom method: asymptotic emmeans(mmodel2, list(pairwise~Ethnicity), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 827711' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 827711' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of Ethnicity`
Ethnicity emmean SE df asymp.LCL asymp.UCL
Asian 63.22 0.1618 Inf 62.90 63.53
Black 61.09 0.1919 Inf 60.72 61.47
Chinese 60.49 0.1771 Inf 60.15 60.84
Mixed 64.10 0.2227 Inf 63.67 64.54
White 65.67 0.1303 Inf 65.42 65.93
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of Ethnicity`
1 estimate SE df z.ratio p.value
Asian - Black 2.125 0.187 Inf 11.391 <.0001
Asian - Chinese 2.722 0.165 Inf 16.481 <.0001
Asian - Mixed -0.887 0.219 Inf -4.042 0.0005
Asian - White -2.457 0.125 Inf -19.692 <.0001
Black - Chinese 0.597 0.197 Inf 3.029 0.0207
Black - Mixed -3.012 0.243 Inf -12.409 <.0001
Black - White -4.582 0.163 Inf -28.163 <.0001
Chinese - Mixed -3.608 0.231 Inf -15.608 <.0001
Chinese - White -5.179 0.145 Inf -35.600 <.0001
Mixed - White -1.570 0.197 Inf -7.985 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
P value adjustment: tukey method for comparing a family of 5 estimates emmeans(mmodel2, list(pairwise~PROGRAMME_PART), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 827711' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 827711' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.$`emmeans of PROGRAMME_PART`
PROGRAMME_PART emmean SE df asymp.LCL asymp.UCL
A 61.98 0.1461 Inf 61.69 62.26
B 62.03 0.1431 Inf 61.75 62.31
C 62.76 0.1646 Inf 62.43 63.08
D 60.78 0.3638 Inf 60.06 61.49
F 67.67 0.5214 Inf 66.64 68.69
T 62.29 0.1372 Inf 62.02 62.56
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Ethnicity
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of PROGRAMME_PART`
1 estimate SE df z.ratio p.value
A - B -0.0492 0.161 Inf -0.305 0.9996
A - C -0.7788 0.190 Inf -4.102 0.0006
A - D 1.2023 0.378 Inf 3.179 0.0185
A - F -5.6881 0.534 Inf -10.649 <.0001
A - T -0.3115 0.194 Inf -1.609 0.5923
B - C -0.7296 0.176 Inf -4.151 0.0005
B - D 1.2515 0.373 Inf 3.355 0.0103
B - F -5.6388 0.534 Inf -10.563 <.0001
B - T -0.2623 0.191 Inf -1.374 0.7426
C - D 1.9811 0.363 Inf 5.450 <.0001
C - F -4.9093 0.540 Inf -9.085 <.0001
C - T 0.4673 0.206 Inf 2.266 0.2083
D - F -6.8904 0.631 Inf -10.923 <.0001
D - T -1.5138 0.379 Inf -3.993 0.0009
F - T 5.3766 0.537 Inf 10.012 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Ethnicity
Degrees-of-freedom method: asymptotic
P value adjustment: tukey method for comparing a family of 6 estimates #There was an interaction so need emmeans for this
emmeans(mmodel2, pairwise ~ Ethnicity | ASSESSMENT_COMPONENT_TYPE_DESC)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 827711' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 827711' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 827711)' or larger];
but be warned that this may result in large computation time and memory use.$emmeans
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
Ethnicity emmean SE df asymp.LCL asymp.UCL
Asian 67.27 0.1633 Inf 66.95 67.59
Black 65.79 0.1939 Inf 65.41 66.17
Chinese 63.84 0.1750 Inf 63.50 64.18
Mixed 68.25 0.2255 Inf 67.81 68.69
White 69.56 0.1309 Inf 69.31 69.82
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
Ethnicity emmean SE df asymp.LCL asymp.UCL
Asian 59.16 0.1719 Inf 58.82 59.49
Black 56.39 0.2066 Inf 55.99 56.80
Chinese 57.15 0.1986 Inf 56.76 57.54
Mixed 59.96 0.2416 Inf 59.48 60.43
White 61.78 0.1343 Inf 61.52 62.05
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$contrasts
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
contrast estimate SE df z.ratio p.value
Asian - Black 1.485 0.190 Inf 7.835 <.0001
Asian - Chinese 3.433 0.163 Inf 21.052 <.0001
Asian - Mixed -0.975 0.223 Inf -4.369 0.0001
Asian - White -2.288 0.127 Inf -18.035 <.0001
Black - Chinese 1.948 0.196 Inf 9.938 <.0001
Black - Mixed -2.459 0.247 Inf -9.971 <.0001
Black - White -3.773 0.165 Inf -22.835 <.0001
Chinese - Mixed -4.408 0.232 Inf -19.025 <.0001
Chinese - White -5.721 0.143 Inf -40.129 <.0001
Mixed - White -1.313 0.200 Inf -6.571 <.0001
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
contrast estimate SE df z.ratio p.value
Asian - Black 2.766 0.208 Inf 13.276 <.0001
Asian - Chinese 2.010 0.196 Inf 10.254 <.0001
Asian - Mixed -0.799 0.244 Inf -3.272 0.0094
Asian - White -2.626 0.139 Inf -18.861 <.0001
Black - Chinese -0.755 0.229 Inf -3.296 0.0087
Black - Mixed -3.564 0.270 Inf -13.203 <.0001
Black - White -5.391 0.181 Inf -29.788 <.0001
Chinese - Mixed -2.809 0.265 Inf -10.614 <.0001
Chinese - White -4.636 0.173 Inf -26.750 <.0001
Mixed - White -1.827 0.219 Inf -8.355 <.0001
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic
P value adjustment: tukey method for comparing a family of 5 estimates Evaluating mixed model 2 assumptions
Homodscedasticity
#plot of mixed model 2
plot(mmodel2)
boxplot(mmodel2@resp$wtres~full_data2$ASSESSMENT_COMPONENT_TYPE_DESC)
boxplot(mmodel2@resp$wtres~full_data2$Ethnicity)
boxplot(mmodel2@resp$wtres~full_data2$PROGRAMME_PART)
Another way to check for homeoscedasticity - (sd) residuals
#ASSESSMENT TYPE
mmodel2res<-mmodel2@resp$wtres
#assessment type residuals in df
mmodel2df<-data.frame(full_data2$ASSESSMENT_COMPONENT_TYPE_DESC,mmodel2res)
names(mmodel2df)<-c("ASSESSMENT_COMPONENT_TYPE_DESC","residual")
mmodel2df %>% group_by(ASSESSMENT_COMPONENT_TYPE_DESC)%>% summarise(sd(residual)) The assumption is met because highest sd is less than twice the lowest.
#Ethnicity
mmodel2df2<-data.frame(full_data2$Ethnicity,mmodel2res)
names(mmodel2df2)<-c("Ethnicity","residual")
mmodel2df2 %>% group_by(Ethnicity)%>% summarise(sd(residual)) Again, the assumption is met.
#Programme part
mmodel2df3<-data.frame(full_data2$PROGRAMME_PART,mmodel2res)
names(mmodel2df3)<-c("Programme_Part","residual")
mmodel2df3 %>% group_by(Programme_Part)%>% summarise(sd(residual))Assumption satisfied again..
Moving onto checking the residuals of the model are normally distributed.
Normality of residuals
#now check for residuals for mmodel2
qqnorm(mmodel2res)
Confidence Intervals
#confint(mmodel2)Research Question 3
3. Is there a link between assessment type, gender and student achievement?
Data set for this question will be full_data data set as already includes Gender data. Created this in preliminary analysis.
3.1 Effect of Gender on student performance
Distribution of Marks of Male & Female students (2017-2021)
boxplot(MARK~Gender, data = full_data)
Below is bar chart showing of Gender of ethnicity on mean mark between 2017-2021
#Visualise effect of ethnicity on overall/mean mark between 2017-2021
full_data %>%
select(Gender, MARK) %>%
group_by(Gender) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
ggplot(aes(x=Gender, y = Average_mark, fill =Gender)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Paired") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Test Scores by Gender between 2017-2021") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Average Mark') 
#looking at avg grade for each ethnicity type between 2017-2021
full_data %>%
select(INSTANCE_ACADEMIC_YEAR, Gender,MARK) %>%
group_by(INSTANCE_ACADEMIC_YEAR, Gender) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
#spread(key = Ethnicity, value = Average_mark) %>%
ggplot(aes(x = INSTANCE_ACADEMIC_YEAR, y = Average_mark, col = Gender)) +
geom_line() +
geom_point() +
ggtitle("Average grade for Men vs Women (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Academic Year') +
ylab('Average Mark') +
labs(col='Gender') 
Analysis of Gender Awarding Gap (2017-2021)
#Visualising percentage of white vs BAME students achieving top degree (2017-2021)
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(MARK >=60) %>%
select(ID2, Gender, Average_mark) %>%
distinct() %>%
group_by(Gender) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Gender, y = counts, fill =Gender)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position = position_stack(vjust = 1.1), size = 3) +
ggtitle("Number of Men vs Women achieving top degree (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
ylab('Number of students')
This is very interesting as more men are being awarded top degrees but on average woman perform better. This may be because more men in this dataset then woman. So, below is a better comparison, looking at distribution of grades.
#Gender AWARDING GAP
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
mutate(Grade = case_when(Average_mark >= 70 ~ "First Class",
Average_mark >= 60 ~ "Upper Second Class Honour",
Average_mark >= 50 ~ "Lower Second Class Honour",
Average_mark >= 40 ~ "Third class/pass",
Average_mark < 40 ~ "Fail")) %>%
group_by(Gender, Grade) %>%
summarise(counts = n()) %>%
mutate(freq = (counts / sum(counts))) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
ggplot(aes(x=Gender, y = freq, fill =Grade)) +
geom_bar(stat = 'identity') +
coord_flip() +
scale_fill_brewer(palette = "Pastel1") +
geom_text(aes(label = percentage), position = position_stack(vjust = 0.5), size = 2.5) +
ggtitle("Distribution of degree outcomes by gender (2017-2021)") +
theme(plot.title = element_text(hjust = 0.5)) +
labs(y = "Frequency", fill = "Degree Classification")`summarise()` has grouped output by 'Gender'. You can override using the `.groups` argument.
This graph is much better at showing what is actually going on in regards to what degree classification students are being awarded. This makes more sense now… This is showing 33.5% of woman students are getting a first in comparison to male students So females do perform better/ more females are awarded a top degree in comparison to male students, but when looking at overall degree outcomes, more males are getting top degree because there are more male students at university.
3.2 Analysing effect of assessment type and gender on student performance and awarding gap
full_data %>%
select(Gender, ASSESSMENT_COMPONENT_TYPE_DESC,MARK) %>%
group_by(Gender, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
tally(mean(MARK)) %>%
rename(Average_mark = n) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = Average_mark, fill = Gender)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Set1") +
geom_text(aes(label = round(Average_mark, 2)), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Average Mark By Assessment Type And Gender") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Average Mark')
This shows women perform better than men in both assessment types, but the difference by assessment type seems similar… So one gender does not perform significantly different than other in one assessment type.
full_data %>%
group_by(ID2) %>%
mutate(Average_mark = mean(MARK)) %>%
filter(Average_mark >=60) %>%
select(ID2, Gender, ASSESSMENT_COMPONENT_TYPE_DESC, Average_mark) %>%
distinct() %>%
select(Gender, ASSESSMENT_COMPONENT_TYPE_DESC) %>%
group_by(Gender,ASSESSMENT_COMPONENT_TYPE_DESC) %>%
summarise(counts = n()) %>%
mutate(percentage = scales::label_percent()(counts / sum(counts))) %>%
#select(Gender, ASSESSMENT_COMPONENT_TYPE_DESC, percentage) %>%
ggplot(aes(x=ASSESSMENT_COMPONENT_TYPE_DESC, y = percentage, fill = Gender)) +
geom_bar(stat = 'identity', position = 'dodge') +
scale_fill_brewer(palette = "Set1") +
geom_text(aes(label = percentage), position=position_dodge(width=0.9), vjust=-0.25) +
ggtitle("Effect of assessment type on gender awarding gap") +
theme(plot.title = element_text(hjust = 0.5)) +
xlab('Assessment Type') +
ylab('Percentage of students')Adding missing grouping variables: `ID2`
`summarise()` has grouped output by 'Gender'. You can override using the `.groups` argument.
5% diff in each assessment type… Doesnt look like exams are leading to a wider gender awarding gap. Difference in performance for exams is same as difference for coursework….
interaction.plot(x.factor = full_data$Gender, #x-axis variable
trace.factor = full_data$ASSESSMENT_COMPONENT_TYPE_DESC, #variable for lines
response = full_data$MARK, #y-axis variable
fun = median, #metric to plot
ylab = "Mark",
xlab = "Ethnicity",
col = c("green", "blue"),
lty = 1, #line type
lwd = 2, #line width
trace.label = "Assessment type")
Statistical Analysis
Mixed Model 3
# **Research Question 3**
# **3. Is there a link between assessment type, gender and student achievement?**
mmodel3<- lmer(MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Gender + PROGRAMME_PART + (1|ID2) + (1|MODULE_CODE), data = full_data)summary(mmodel3)Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: MARK ~ ASSESSMENT_COMPONENT_TYPE_DESC * Gender + PROGRAMME_PART +
(1 | ID2) + (1 | MODULE_CODE)
Data: full_data
REML criterion at convergence: 6689154
Scaled residuals:
Min 1Q Median 3Q Max
-7.4574 -0.5327 0.0223 0.5787 4.7917
Random effects:
Groups Name Variance Std.Dev.
ID2 (Intercept) 40.57 6.369
MODULE_CODE (Intercept) 25.44 5.044
Residual 153.51 12.390
Number of obs: 840368, groups: ID2, 31540; MODULE_CODE, 6945
Fixed effects:
Estimate Std. Error df t value
(Intercept) 6.729e+01 1.431e-01 1.236e+04 470.102
ASSESSMENT_COMPONENT_TYPE_DESCExam -8.194e+00 4.717e-02 7.750e+05 -173.732
GenderWoman 8.882e-01 8.896e-02 3.505e+04 9.985
PROGRAMME_PARTB 2.593e-02 1.613e-01 1.548e+04 0.161
PROGRAMME_PARTC 7.156e-01 1.899e-01 1.292e+04 3.767
PROGRAMME_PARTD -1.454e+00 3.780e-01 3.142e+04 -3.846
PROGRAMME_PARTF 5.782e+00 5.344e-01 6.410e+03 10.819
PROGRAMME_PARTT -1.546e+00 1.887e-01 1.131e+04 -8.194
ASSESSMENT_COMPONENT_TYPE_DESCExam:GenderWoman 8.917e-01 6.904e-02 8.360e+05 12.916
Pr(>|t|)
(Intercept) < 2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam < 2e-16 ***
GenderWoman < 2e-16 ***
PROGRAMME_PARTB 0.872332
PROGRAMME_PARTC 0.000166 ***
PROGRAMME_PARTD 0.000120 ***
PROGRAMME_PARTF < 2e-16 ***
PROGRAMME_PARTT 2.8e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESCExam:GenderWoman < 2e-16 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation of Fixed Effects:
(Intr) ASSESSMENT_COMPONENT_TYPE_DESCEx GndrWm
ASSESSMENT_COMPONENT_TYPE_DESCEx -0.117
GenderWoman -0.245 0.104
PROGRAMME_PARTB -0.580 -0.005 -0.003
PROGRAMME_PARTC -0.557 0.000 0.008
PROGRAMME_PARTD -0.297 0.006 0.014
PROGRAMME_PARTF -0.230 0.005 0.014
PROGRAMME_PARTT -0.696 0.036 -0.023
ASSESSMENT_COMPONENT_TYPE_DESCE: 0.048 -0.467 -0.209
PROGRAMME_PARTB PROGRAMME_PARTC PROGRAMME_PARTD
ASSESSMENT_COMPONENT_TYPE_DESCEx
GenderWoman
PROGRAMME_PARTB
PROGRAMME_PARTC 0.510
PROGRAMME_PARTD 0.246 0.328
PROGRAMME_PARTF 0.153 0.144 0.075
PROGRAMME_PARTT 0.444 0.430 0.255
ASSESSMENT_COMPONENT_TYPE_DESCE: 0.003 0.000 -0.001
PROGRAMME_PARTF PROGRAMME_PARTT
ASSESSMENT_COMPONENT_TYPE_DESCEx
GenderWoman
PROGRAMME_PARTB
PROGRAMME_PARTC
PROGRAMME_PARTD
PROGRAMME_PARTF
PROGRAMME_PARTT 0.170
ASSESSMENT_COMPONENT_TYPE_DESCE: -0.002 0.003 This is interesting…shows gap in AT smaller for woman compared to men. Also average cw grade very similar for men and woman And woman perform better in exams compared to men And woman perform better in exams compared to men…
plot_model(mmodel3, type = "pred", terms = c("Gender", "ASSESSMENT_COMPONENT_TYPE_DESC"))
anova(mmodel3)Type III Analysis of Variance Table with Satterthwaite's method
Sum Sq Mean Sq NumDF DenDF F value Pr(>F)
ASSESSMENT_COMPONENT_TYPE_DESC 4858953 4858953 1 716363 31652.242 < 2.2e-16 ***
Gender 34941 34941 1 34071 227.611 < 2.2e-16 ***
PROGRAMME_PART 44305 8861 5 12859 57.723 < 2.2e-16 ***
ASSESSMENT_COMPONENT_TYPE_DESC:Gender 25610 25610 1 835965 166.826 < 2.2e-16 ***
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1Again, similar to RQ1 & RQ2, all variables hold a significant effect on student performance
#effect size...look at video for how to explain this in report.
effectsize::eta_squared(mmodel3, partial = TRUE)# Effect Size for ANOVA (Type III)
Parameter | Eta2 (partial) | 95% CI
---------------------------------------------------------------------
ASSESSMENT_COMPONENT_TYPE_DESC | 0.04 | [0.05, 1.00]
Gender | 6.64e-03 | [0.01, 1.00]
PROGRAMME_PART | 0.02 | [0.02, 1.00]
ASSESSMENT_COMPONENT_TYPE_DESC:Gender | 2.00e-04 | [0.00, 1.00]
- One-sided CIs: upper bound fixed at [1.00].#emmeans is meant to be looked at for main effect.
#in this case they were all significant so will do for each??
emmeans(mmodel3, list(pairwise~ASSESSMENT_COMPONENT_TYPE_DESC), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of ASSESSMENT_COMPONENT_TYPE_DESC`
ASSESSMENT_COMPONENT_TYPE_DESC emmean SE df asymp.LCL asymp.UCL
Coursework 68.32 0.1268 Inf 68.07 68.57
Exam 60.57 0.1297 Inf 60.32 60.83
Results are averaged over the levels of: Gender, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of ASSESSMENT_COMPONENT_TYPE_DESC`
1 estimate SE df z.ratio p.value
Coursework - Exam 7.75 0.0436 Inf 177.911 <.0001
Results are averaged over the levels of: Gender, PROGRAMME_PART
Degrees-of-freedom method: asymptotic emmeans(mmodel3, list(pairwise~Gender), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
NOTE: Results may be misleading due to involvement in interactions$`emmeans of Gender`
Gender emmean SE df asymp.LCL asymp.UCL
Man 63.78 0.1301 Inf 63.53 64.04
Woman 65.12 0.1376 Inf 64.85 65.38
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of Gender`
1 estimate SE df z.ratio p.value
Man - Woman -1.33 0.0884 Inf -15.087 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, PROGRAMME_PART
Degrees-of-freedom method: asymptotic emmeans(mmodel3, list(pairwise~PROGRAMME_PART), adjust="tukey")Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.$`emmeans of PROGRAMME_PART`
PROGRAMME_PART emmean SE df asymp.LCL asymp.UCL
A 63.86 0.1384 Inf 63.59 64.13
B 63.89 0.1354 Inf 63.62 64.15
C 64.58 0.1584 Inf 64.27 64.89
D 62.41 0.3612 Inf 61.70 63.12
F 69.64 0.5200 Inf 68.62 70.66
T 62.31 0.1309 Inf 62.06 62.57
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Gender
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$`pairwise differences of PROGRAMME_PART`
1 estimate SE df z.ratio p.value
A - B -0.0259 0.161 Inf -0.161 1.0000
A - C -0.7155 0.190 Inf -3.767 0.0023
A - D 1.4538 0.378 Inf 3.846 0.0017
A - F -5.7823 0.534 Inf -10.819 <.0001
A - T 1.5463 0.189 Inf 8.194 <.0001
B - C -0.6896 0.176 Inf -3.927 0.0012
B - D 1.4797 0.373 Inf 3.969 0.0010
B - F -5.7563 0.534 Inf -10.777 <.0001
B - T 1.5723 0.186 Inf 8.450 <.0001
C - D 2.1693 0.363 Inf 5.974 <.0001
C - F -5.0667 0.541 Inf -9.370 <.0001
C - T 2.2619 0.202 Inf 11.187 <.0001
D - F -7.2361 0.631 Inf -11.469 <.0001
D - T 0.0925 0.377 Inf 0.245 0.9999
F - T 7.3286 0.536 Inf 13.681 <.0001
Results are averaged over the levels of: ASSESSMENT_COMPONENT_TYPE_DESC, Gender
Degrees-of-freedom method: asymptotic
P value adjustment: tukey method for comparing a family of 6 estimates #There was an interaction so need emmeans for this
emmeans(mmodel3, pairwise ~ Gender | ASSESSMENT_COMPONENT_TYPE_DESC)Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'pbkrtest.limit = 840368' (or larger)
[or, globally, 'set emm_options(pbkrtest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.
Note: D.f. calculations have been disabled because the number of observations exceeds 3000.
To enable adjustments, add the argument 'lmerTest.limit = 840368' (or larger)
[or, globally, 'set emm_options(lmerTest.limit = 840368)' or larger];
but be warned that this may result in large computation time and memory use.$emmeans
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
Gender emmean SE df asymp.LCL asymp.UCL
Man 67.88 0.1307 Inf 67.62 68.13
Woman 68.77 0.1380 Inf 68.50 69.04
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
Gender emmean SE df asymp.LCL asymp.UCL
Man 59.68 0.1337 Inf 59.42 59.95
Woman 61.46 0.1442 Inf 61.18 61.75
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic
Confidence level used: 0.95
$contrasts
ASSESSMENT_COMPONENT_TYPE_DESC = Coursework:
contrast estimate SE df z.ratio p.value
Man - Woman -0.888 0.089 Inf -9.985 <.0001
ASSESSMENT_COMPONENT_TYPE_DESC = Exam:
contrast estimate SE df z.ratio p.value
Man - Woman -1.780 0.101 Inf -17.704 <.0001
Results are averaged over the levels of: PROGRAMME_PART
Degrees-of-freedom method: asymptotic Evaluating mixed model-3 assumptions
Do not need to check linearity as each variable is categorical.
Homodscedasticity
#plot of mixed model 3 residuals
plot(mmodel3)
boxplot(mmodel3@resp$wtres~full_data$ASSESSMENT_COMPONENT_TYPE_DESC)
boxplot(mmodel3@resp$wtres~full_data$Gender)
boxplot(mmodel3@resp$wtres~full_data$PROGRAMME_PART)
Another way to check for homeoscedasticity - (sd) residuals
#ASSESSMENT TYPE
mmodel3res<-mmodel3@resp$wtres
#assessment type residuals in df
mmodel3df<-data.frame(full_data$ASSESSMENT_COMPONENT_TYPE_DESC,mmodel3res)
names(mmodel3df)<-c("ASSESSMENT_COMPONENT_TYPE_DESC","residual")
mmodel3df %>% group_by(ASSESSMENT_COMPONENT_TYPE_DESC)%>% summarise(sd(residual))#Gender
mmodel3df2<-data.frame(full_data$Gender,mmodel3res)
names(mmodel3df2)<-c("Gender","residual")
mmodel3df2 %>% group_by(Gender)%>% summarise(sd(residual)) #Programme part
mmodel3df3<-data.frame(full_data$PROGRAMME_PART,mmodel3res)
names(mmodel3df3)<-c("Programme_Part","residual")
mmodel3df3 %>% group_by(Programme_Part)%>% summarise(sd(residual))Assumptions met for all variables
Normality of residuals
qqnorm(mmodel3res)
All assumptions for mixed model 3 met.
Confidence Intervals
#confint(mmodel3)LS0tCnRpdGxlOiAiRGF0YSBzdW1tYXJ5IGFuZCBzdGF0aXN0aWNhbCBhbmFseXNpcyBub3RlYm9vayIKYXV0aG9yOiAiU3ViZWlkYSBIYXNzYW4iCmRhdGU6ICIyNnRoIEF1Z3VzdCAyMDIyIgpvdXRwdXQ6CiAgaHRtbF9kb2N1bWVudDoKICAgIHRvYzogeWVzCiAgICB0b2NfZGVwdGg6ICczJwogICAgZGZfcHJpbnQ6IHBhZ2VkCiAgcGRmX2RvY3VtZW50OgogICAgdG9jOiB5ZXMKICAgIHRvY19kZXB0aDogMwogIGh0bWxfbm90ZWJvb2s6CiAgICBzZWxmX2NvbnRhaW5lZDogeWVzCiAgICBoaWdobGlnaHQ6IHRleHRtYXRlCiAgICB0b2M6IHllcwogICAgdG9jX2RlcHRoOiAzCiAgICBudW1iZXJfc2VjdGlvbnM6IG5vCi0tLQoKYGBge3Igc2V0dXAsIGluY2x1ZGUgPSBGQUxTRX0Ka25pdHI6Om9wdHNfY2h1bmskc2V0KAogIGNvbW1lbnQgPSAiIiwKICByZXN1bHRzID0gImhvbGQiLAogIGVjaG8gPSBUUlVFLAogIG1lc3NhZ2UgPSBGQUxTRSwKICB3YXJuaW5nID0gRkFMU0UsCiAgZmlnLndpZHRoID0gNCwKICBmaWcuaGVpZ2h0ID0gMi42LAogIGZpZy5hbGlnbiA9ICJjZW50ZXIiCikgCmBgYAoKTG9hZGluZyByZXF1aXJlZCBwYWNrYWdlcwpgYGB7ciwgbWVzc2FnZSA9IEZBTFNFLGNvbW1lbnRzPUZBTFNFLCB3YXJuaW5nID0gRkFMU0V9CiNpbXBvcnRpbmcgbGlicmFyaWVzCmxpYnJhcnkoInRpZHl2ZXJzZSIpCmxpYnJhcnkoaGVyZSkKbGlicmFyeShnZ3JlcGVsKQpsaWJyYXJ5KGxtZTQpCmxpYnJhcnkobG1lclRlc3QpCmxpYnJhcnkoc2pzdGF0cykKbGlicmFyeShlbW1lYW5zKQpsaWJyYXJ5KE11TUluKQpsaWJyYXJ5KHNqUGxvdCkKbGlicmFyeShzam1pc2MpCmxpYnJhcnkoImNhciIpCmBgYAoKIyAqKlByZWxpbWluYXJ5IGFuYWx5c2lzKioKUmVhZGluZyBkYXRhIGludG8gZGF0YSBmcmFtZQpgYGB7cn0KI0V0aG5pY2l0eSBkYXRhIHNldApldGhuaWNpdHlfZGF0YSA8LSByZWFkX2RlbGltKGZpbGUgPSBoZXJlKCJkYXRhIiwgInFyeV9FdGhuaWNfb3JpZ2luXzIudHh0IiksIGRlbGltID0gIiwiKQpoZWFkKGV0aG5pY2l0eV9kYXRhKSAjcHJldmlld2luZyBmaXJzdCA1IGxpbmVzCgojTWFyayBkYXRhIHNldAptYXJrX2RhdGEgPC0gcmVhZF9kZWxpbShmaWxlID0gaGVyZSgiZGF0YSIsICJxcnlfTWFya19EYXRhXzIudHh0IiksIGRlbGltID0gIiwiKQpoZWFkKG1hcmtfZGF0YSkgI3ByZXZpZXdpbmcgZmlyc3QgNSBsaW5lcwpgYGAKCiMjIEV0aG5pY2l0eSBEYXRhc2V0CiMjIyBDbGVhbmluZyAKYGBge3J9CiNjaGVja2luZyBmb3IgbWlzc2luZyBkYXRhCmV0aG5pY2l0eV9kYXRhICU+JSBzYXBwbHkoZnVuY3Rpb24oeCkgc3VtKGlzLm5hKHgpKSApCgojcmVtb3ZpbmcgbWlzc2luZyBkYXRhCmNsZWFuX2V0aG5pY2l0eV9kYXRhIDwtIGV0aG5pY2l0eV9kYXRhICU+JQogIGRyb3BfbmEoKQoKI2NoZWNraW5nIGlmIG5hJ3MgcmVtb3ZlZC4KY2xlYW5fZXRobmljaXR5X2RhdGEgJT4lIHNhcHBseShmdW5jdGlvbih4KSBzdW0oaXMubmEoeCkpKQoKYGBgCgpgYGB7cn0KI2NoZWNraW5nIGZvciBkdXBsaWNhdGVzICYgcmVtb3ZpbmcKbGVuZ3RoKHVuaXF1ZShjbGVhbl9ldGhuaWNpdHlfZGF0YSRJRDIpKSAjbnVtYmVyIG9mIHN0dWRlbnRzIGluIGV0aG5pYyBkYXRhCmxlbmd0aCh1bmlxdWUoY2xlYW5fZXRobmljaXR5X2RhdGEkSUQyKSkgPT0gbnJvdyhjbGVhbl9ldGhuaWNpdHlfZGF0YSkgI3JldHVybnMgVFJVRSBhcyBubyBkdXBsaWNhdGVzCgpgYGAKCmBgYHtyfQojcmVuYW1pbmcgY29sdW1ucyAmIHNhdmluZyBpbiBuZXcgZGF0YSBmcmFtZSAKY2xlYW5fZXRobmljaXR5X2RhdGEgJT4lCiAgcmVuYW1lKAogICAgR2VuZGVyID0gR0VOREVSX0RFU0MsCiAgICBFdGhuaWNpdHkgPSBFVEhOSUNfT1JJR0lOX0RFU0MKICAgICkgLT4gY2xlYW5fZXRobmljaXR5X2RhdGEKYGBgCgpgYGB7cn0KI0NvbXB1dGluZyBwZXJjZW50YWdlIG9mIHN0dWRlbnRzIGJlbG9uZ2luZyB0byBlYWNoIGdlbmRlci9ldGhuaWMgZ3JvdXAKCmNsZWFuX2V0aG5pY2l0eV9kYXRhICU+JQogIGdyb3VwX2J5KEdlbmRlcikgJT4lCiAgdGFsbHkgKCkgJT4lCiAgbXV0YXRlKGZyZXEgPSAobiAvIHN1bShuKSkqMTAwKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKG4gLyBzdW0obikpKSAlPiUKICByZW5hbWUoIk51bWJlciBvZiBzdHVkZW50cyIgPSBuKQoKI0NhdGVnb3Jpc2luZyBpbnRvIG1hbiB2cyB3b21hbiBvdGhlciBsZXZlbHMgdG8gbGl0dGxlIGNhc2VzCiNyZW1vdmUgbm9uIGJpbmFyeSwgb3RoZXIgJiBwcmVmZXIgbm90IHRvIHNheQpjbGVhbl9ldGhuaWNpdHlfZGF0YTwtY2xlYW5fZXRobmljaXR5X2RhdGFbIShjbGVhbl9ldGhuaWNpdHlfZGF0YSRHZW5kZXI9PSJOb24tQmluYXJ5IiB8IGNsZWFuX2V0aG5pY2l0eV9kYXRhJEdlbmRlcj09IlByZWZlciBub3QgdG8gc2F5IiB8IGNsZWFuX2V0aG5pY2l0eV9kYXRhJEdlbmRlcj09Ik90aGVyIikgLF0KCgojY2hlY2sgbGV2ZWxzIG9mIGV0aG5pY2l0ZXMKY2xlYW5fZXRobmljaXR5X2RhdGEgJT4lCiAgZ3JvdXBfYnkoRXRobmljaXR5KSAlPiUKICB0YWxseSAoKSAlPiUKICBtdXRhdGUoZnJlcSA9IChuIC8gc3VtKG4pKSoxMDApICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkobiAvIHN1bShuKSkpCmBgYAoKSSB3aWxsIGNhdGVnb3Jpc2UgZXRobmljaXR5IGRpZmZlcmVudGx5IGRlcGVuZGluZyBvbiByZXNlYXJjaCBxdWVzdGlvbi4gRmlyc3RseSwgSSB3aWxsIGJlZ2luIHdpdGggd2hpdGUgdnMgb3RoZXIuCgpgYGB7cn0KI2R1cGxpY2F0aW5nIGRhdGEgc2V0IGF0IHRoaXMgcG9pbnQgYXMgd2lsbCBjYXRlZ29yaXNlIGRpZmZlcmVudGx5IGZvciBzZWNvbmQgcmVzZWFyY2ggcXVlc3Rpb24KY2xlYW5fZXRobmljaXR5X2RhdGEgLT4gY2xlYW5fZXRobmljaXR5X2RhdGEyCmBgYAoKYGBge3J9CiNjYXRlZ29yaXNpbmcgZXRobmljaXRpZXMgaW50byB3aGl0ZSB2cyBub24gd2hpdGUKI3JlbmFtZSBhbGwgd2hpdGUgY29sdW1ucy0gd2hpdGUuCiNyZW5hbWUgYWxsIG90aGVyIGNvbHVtbnMgb3RoZXIuCgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiV2hpdGUgLSBCcml0aXNoIl0gPC0gIldoaXRlIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiV2hpdGUgLSBJcmlzaCJdIDwtICJXaGl0ZSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIldoaXRlIC0gU2NvdHRpc2giXSA8LSAiV2hpdGUiCmNsZWFuX2V0aG5pY2l0eV9kYXRhW2NsZWFuX2V0aG5pY2l0eV9kYXRhID09ICJPdGhlciBXaGl0ZSBCYWNrZ3JvdW5kIl0gPC0gIldoaXRlIgoKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIkFyYWIiXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIkFzaWFuIE9yIEFzaWFuIEJyaXRpc2ggLSBCYW5nbGFkZXNoaSJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiQXNpYW4gT3IgQXNpYW4gQnJpdGlzaCAtIEluZGlhbiJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiQXNpYW4gT3IgQXNpYW4gQnJpdGlzaCAtIFBha2lzdGFuaSJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiQmxhY2sgT3IgQmxhY2sgQnJpdGlzaCAtIEFmcmljYW4iXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIkJsYWNrIE9yIEJsYWNrIEJyaXRpc2ggLSBDYXJpYmJlYW4iXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIkNoaW5lc2UiXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIkd5cHN5IG9yIFRyYXZlbGxlciJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiTWl4ZWQgLSBXaGl0ZSBBbmQgQXNpYW4iXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIk1peGVkIC0gV2hpdGUgQW5kIEJsYWNrIEFmcmljYW4iXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIk1peGVkIC0gV2hpdGUgQW5kIEJsYWNrIENhcmliYmVhbiJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiT3RoZXIgQXNpYW4gQmFja2dyb3VuZCJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiT3RoZXIgQmxhY2sgQmFja2dyb3VuZCJdIDwtICJCQU1FIgpjbGVhbl9ldGhuaWNpdHlfZGF0YVtjbGVhbl9ldGhuaWNpdHlfZGF0YSA9PSAiT3RoZXIgRXRobmljIEJhY2tncm91bmQiXSA8LSAiQkFNRSIKY2xlYW5fZXRobmljaXR5X2RhdGFbY2xlYW5fZXRobmljaXR5X2RhdGEgPT0gIk90aGVyIE1peGVkIEJhY2tncm91bmQiXSA8LSAiQkFNRSIKCiNyZW1vdmUgbm90IGtub3duLG5vIGluZm9ybWF0aW9uICYgaW5mbyByZWZ1c2VkLSBhcyBjb3VsZCBiZSBhbnkgZXRobmljaXR5LgoKY2xlYW5fZXRobmljaXR5X2RhdGE8LWNsZWFuX2V0aG5pY2l0eV9kYXRhWyEoY2xlYW5fZXRobmljaXR5X2RhdGEkRXRobmljaXR5PT0iSW5mb3JtYXRpb24gUmVmdXNlZCIgfCBjbGVhbl9ldGhuaWNpdHlfZGF0YSRFdGhuaWNpdHk9PSJJbmZvcm1hdGlvbiBSZWZ1c2VkIC0gT0JTT0xFVEUgVkFMVUUiIHwgY2xlYW5fZXRobmljaXR5X2RhdGEkRXRobmljaXR5PT0iTm8gSW5mb3JtYXRpb24iIHwgY2xlYW5fZXRobmljaXR5X2RhdGEkRXRobmljaXR5PT0iTm90IEtub3duIikgLF0KCmBgYAoKYGBge3J9CiNjaGFuZ2luZyBudW1lcmljYWwgdmFyaWFibGVzIHRvIGZhY3RvcnMuIApjbGVhbl9ldGhuaWNpdHlfZGF0YSAlPiUKICBtdXRhdGVfYXQodmFycyhHZW5kZXIsIEV0aG5pY2l0eSksIGxpc3QoZmFjdG9yKSkgLT4gY2xlYW5fZXRobmljaXR5X2RhdGEKCnN1bW1hcnkoY2xlYW5fZXRobmljaXR5X2RhdGEpCgpgYGAKCiMjIyBWaXN1YWxpc2F0aW9uCmBgYHtyfQojRXRobmljaXR5CiNWaXN1YWxpc2luZyBudW1iZXIgb2Ygc3R1ZGVudHMgaW4gZWFjaCBldGhuaWNpdHkgY2F0ZWdvcnkgCgpldGhuaWNpdHlfZGF0YSAlPiUKICBncm91cF9ieShFVEhOSUNfT1JJR0lOX0RFU0MpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShmcmVxID0gKGNvdW50cyAvIHN1bShjb3VudHMpKSoxMDApICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoLiwgYWVzKHggPSBFVEhOSUNfT1JJR0lOX0RFU0MsIHkgPSBjb3VudHMpKSArCiAgZ2VvbV9iYXIoZmlsbCA9ICIjMDA3M0MyRkYiLCBzdGF0ID0gImlkZW50aXR5IikgKwogICNnZW9tX3RleHQoYWVzKGxhYmVsID0gcGVyY2VudGFnZSksIGFuZ2xlID0gNjApICsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDYwLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBoanVzdCA9IDEpKSArCiAgZ2d0aXRsZSgiTnVtYmVyIG9mIHN0dWRlbnRzIGluIGVhY2ggZXRobmljIGNhdGVnb3J5IikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeGxhYignRXRobmljaXR5JykgKwogIHlsYWIoJ051bWJlciBvZiBzdHVkZW50cycpIApgYGAKCmBgYHtyfQojRXRobmljaXR5CiNQZXJjZW50YWdlIG9mIHN0dWRlbnRzIGJlbG9uZ2luZyB0byBlYWNoIGV0aG5pY2l0eSBhZnRlciBjYXRlZ29yaXNhdGlvbgpjbGVhbl9ldGhuaWNpdHlfZGF0YSAlPiUKICBncm91cF9ieShFdGhuaWNpdHkpICU+JQogIHRhbGx5ICgpICU+JQogIG11dGF0ZShmcmVxID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkobiAvIHN1bShuKSkpICAlPiUKICBnZ3Bsb3QoIC4sIGFlcyh4PSIiLCB5PWZyZXEsIGZpbGw9RXRobmljaXR5KSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5Iiwgd2lkdGg9MSwgY29sb3I9IndoaXRlIikgKwogIGNvb3JkX3BvbGFyKCJ5Iiwgc3RhcnQ9MCkgKwogIHRoZW1lX3ZvaWQoKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IGZyZXEpLCBjb2xvciA9ICJ3aGl0ZSIsIHBvc2l0aW9uID0gcG9zaXRpb25fc3RhY2sodmp1c3QgPSAwLjUpLCBzaXplID0gNCkgKwogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiUGFpcmVkIikgKwogIGdndGl0bGUoIlBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgaW4gZWFjaCBldGhuaWMgY2F0ZWdvcnkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpIApgYGAKCmBgYHtyfQojR0VOREVSCiNQZXJjZW50YWdlIG9mIHN0dWRlbnRzIGJlbG9uZ2luZyB0byBlYWNoIGdlbmRlciBjYXRlZ29yeQpldGhuaWNpdHlfZGF0YSAlPiUKICBncm91cF9ieShHRU5ERVJfREVTQykgJT4lCiAgc3VtbWFyaXNlKGNvdW50cyA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSAoY291bnRzIC8gc3VtKGNvdW50cykpKjEwMCkgJT4lCiAgbXV0YXRlKHBlcmNlbnRhZ2UgPSBzY2FsZXM6OmxhYmVsX3BlcmNlbnQoKShjb3VudHMgLyBzdW0oY291bnRzKSkpICU+JQogIGdncGxvdCguLCBhZXMoeCA9IEdFTkRFUl9ERVNDLCB5ID0gY291bnRzLCBmaWxsPUdFTkRFUl9ERVNDKSkgKwogIGdlb21fYmFyKHN0YXQgPSAiaWRlbnRpdHkiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHBlcmNlbnRhZ2UpLCB2anVzdCA9IC0wLjUpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYWlyZWQiKSArCiAgZ2d0aXRsZSgiUGVyY2VudGFnZSBvZiBzdHVkZW50cyBpbiBlYWNoIGdlbmRlciBjYXRlZ29yeSIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ0dlbmRlcicpICsKICB5bGFiKCdOdW1iZXIgb2Ygc3R1ZGVudHMnKSArCiAgZ3VpZGVzKGZpbGwgPSBndWlkZV9sZWdlbmQodGl0bGUgPSAiR2VuZGVyIERlc2NyaXB0aW9uICIpKSAKYGBgCgpgYGB7cn0KI0dFTkRFUgojUGVyY2VudGFnZSBvZiBzdHVkZW50cyBiZWxvbmdpbmcgdG8gZWFjaCBnZW5kZXIgYWZ0ZXIgY2F0ZWdvcmlzYXRpb24KY2xlYW5fZXRobmljaXR5X2RhdGEgJT4lCiAgZ3JvdXBfYnkoR2VuZGVyKSAlPiUKICB0YWxseSAoKSAlPiUKICBtdXRhdGUoZnJlcSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKG4gLyBzdW0obikpKSAlPiUKICBnZ3Bsb3QoIC4sIGFlcyh4PSIiLCB5PWZyZXEsIGZpbGw9R2VuZGVyKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5Iiwgd2lkdGg9MSwgY29sb3I9IndoaXRlIikgKwogIGNvb3JkX3BvbGFyKCJ5Iiwgc3RhcnQ9MCkgKwogIHRoZW1lX3ZvaWQoKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IGZyZXEpLCBjb2xvciA9ICJ3aGl0ZSIsIHBvc2l0aW9uID0gcG9zaXRpb25fc3RhY2sodmp1c3QgPSAwLjUpLCBzaXplID0gNSkgKwogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiU2V0MiIpICsKICBnZ3RpdGxlKCJQZXJjZW50YWdlIG9mIHN0dWRlbnRzIGluIGVhY2ggZ2VuZGVyIGNhdGVnb3J5IikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSAKYGBgCgojIyBNYXJrIERhdGFzZXQKIyMjIENsZWFuaW5nIApgYGB7cn0KI3N1bW1hcnkgb2YgbWFyayBkYXRhCnN1bW1hcnkobWFya19kYXRhKQpgYGAKCmBgYHtyfQojY2hlY2tpbmcgZm9yIG1pc3NpbmcgZGF0YT8KbWFya19kYXRhICU+JSBzYXBwbHkoZnVuY3Rpb24oeCkgc3VtKGlzLm5hKHgpKSApCgojcmVtb3ZlIG1pc3NpbmcgZGF0YT8gLSBkZWNpZGVkIHRvIG5vdCByZW1vdmUgbWlzc2luZyBkYXRhIGFzIG9ubHkgbWlzc2luZyBkYXRhIGluIHByb2dyYW1tZSBzY2hvb2wsIG1vZHVsZSBzY2hvb2wgYW5kIHBhcnRfZW5kX2RhdGUgYW5kIG1vZHVsZSBtYXJrLSBpIGFtIG5vdCBkaXJlY3RseSBhc3Nlc3NpbmcgdGhlc2UgY29sdW1ucy0gc28gZG9lc24ndCBtYXR0ZXIgdG9vIG11Y2guIElmIGxvb2tpbmcgYXQgZWZmZWN0IG9mIHNjaG9vbCBvbiBtYXJrIG1heSBuZWVkIHRvIGNvbnNpZGVyIHJlbW92aW5nIG1pc3NpbmcgdmFsdWVzLiAKCiNjaGVja2luZyBmb3IgZHVwbGljYXRlcwpsZW5ndGgodW5pcXVlKG1hcmtfZGF0YSRJRDIpKSAjbnVtYmVyIG9mIHN0dWRlbnRzIGluIG1hcmtzIGRhdGEtIHNhbWUgYXMgZXRobmljaXR5IGRhdGEgc2V0Cmxlbmd0aCh1bmlxdWUobWFya19kYXRhJElEMikpID09IG5yb3cobWFya19kYXRhKSAjcmV0dXJucyBGQUxTRSBhcyB0aGVyZSBhcmUgZHVwbGljYXRlcwoKI3JlbW92aW5nIGR1cGxpY2F0ZSByb3dzLSAxLDExNCw5MTEgcm93cyBvbmNlIGR1cGxpY2F0ZXMgcmVtb3ZlZC4KZGlzdGluY3QobWFya19kYXRhKSAtPiBjbGVhbl9tYXJrX2RhdGEKYGBgCgpgYGB7cn0KI2NoYW5naW5nIHZhcmlhYmxlcyB0byBmYWN0b3JzLgpjbGVhbl9tYXJrX2RhdGEgJT4lCiAgbXV0YXRlX2F0KHZhcnMoUFJPR1JBTU1FX1NDSE9PTCwgUFJPR1JBTU1FX09XTkVSLCBQUk9HUkFNTUVfTEVWRUxfREVTQywgUFJPR1JBTU1FX1BBUlQsIFBBUlRfRU5EX0RBVEUsIE1PRFVMRV9PV05FUiwgTU9EVUxFX1NDSE9PTCwgTU9EVUxFX0NPREUsIE1PRFVMRV9MT05HX1RJVExFLCBJTkNMVURFX01BUktfREVTQywgQVNTRVNTTUVOVF9DT01QT05FTlRfQ09ERSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVElUTEUsIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyksIGxpc3QoZmFjdG9yKSkgLT4gY2xlYW5fbWFya19kYXRhCmBgYAoKCiMjIyBWaXN1YWxpc2F0aW9uCmBgYHtyfQojaG93IG1hbnkgeWVhcnMgb2YgZGF0YT8KY2xlYW5fbWFya19kYXRhICU+JQogIGdyb3VwX2J5KFNUVURFTlRfU1RBUlRfWUVBUikgJT4lCiAgY291bnQgKCkKI2RhdGEgc3BhbnMgMjAwNy0yMDIyCiNPbmx5IGludGVyZXN0ZWQgaW4gdGhlIGxhc3QgNSB5ZWFycywgYXMgYmVmb3JlIDIwMTYsIGEgbG90IGxlc3MgZGF0YS9vdXRkYXRlZC4KI21hcmsgZGF0YSBvbmx5IGhhcyBhIHNpbmdsZSBzdHVkZW50cyByZXN1bHRzIGZvciAyMDIyLCB0aGVyZWZvcmUsIG9ubHkgb25lIGdlbmRlciwgb25lIGV0aG5pY2l0eS4KI1RoaXMgaXMgbm90IGdvb2QgZW5vdWdoIGZvciBhbnkgYW5hbHlzaXMgb24gc3R1ZGVudHMgcGVyZm9ybWFuY2UsIHdpbGwgY2F1c2Ugc2VyaW91cyBvdmVyIGdlbmVyYWxpc2F0aW9uLgoKCiNyZW1vdmUgZGF0YSBiZWZvcmUgMjAxNyAmIGFmdGVyIDIwMjEKY2xlYW5fbWFya19kYXRhICU+JQogIGZpbHRlcihTVFVERU5UX1NUQVJUX1lFQVIgPj0gMTcgJiBTVFVERU5UX1NUQVJUX1lFQVIgPD0gMjEpIC0+Y2xlYW5fbWFya19kYXRhCgojVmlzdWFsaXNhdGlvbiBvZiBzdHVkZW50cyBpbiBlYWNoIHllYXIgZ3JvdXAuCmNsZWFuX21hcmtfZGF0YSAlPiUKICBncm91cF9ieShTVFVERU5UX1NUQVJUX1lFQVIpICU+JSAKICBjb3VudCgpICU+JQogIGdncGxvdCguLCBhZXMoeCA9IFNUVURFTlRfU1RBUlRfWUVBUiwgeT0gbikpICsgCiAgZ2VvbV9iYXIoc3RhdCA9ICJpZGVudGl0eSIsIGZpbGwgPSAiZGFya2dyZWVuIiwgc2l6ZSA9IDIuNSkgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBuKSwgdmp1c3QgPSAtMC41KSArCiAgZ2d0aXRsZSgiTnVtYmVyIG9mIHN0dWRlbnRzIGluIGVhY2ggeWVhciAoMjAxNy0yMDIxKSIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ1N0dWRlbnQgU3RhcnQgWWVhcicpICsKICB5bGFiKCdOdW1iZXIgb2Ygc3R1ZGVudHMnKSAgCmBgYAoKYGBge3J9CiNIb3cgbWFueSBhc3Nlc3NtZW50IHR5cGVzIGluIGRhdGEgc2V0PwpjbGVhbl9tYXJrX2RhdGEgJT4lCiAgZ3JvdXBfYnkoQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBjb3VudCgpCiNDb3Vyc2V3b3JrLCBFeGFtLCBJbi1DbGFzcyBUZXN0LCBJbi1EZXB0IEV4YW0sIExhYm9yYXRvcnkgVGVzdAoKI3R5cGVzIG9mIGFzc2Vzc21lbnQgdHlwZXMgJiByZWxhdGl2ZSBwZXJjZW50YWdlcyBvZiBzdHVkZW50cyB3aG8gZGlkIHRoZSBBVApjbGVhbl9tYXJrX2RhdGEgJT4lCiAgZ3JvdXBfYnkoQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBzdW1tYXJpc2UoY291bnRzID0gbigpKSAlPiUKICBtdXRhdGUoZnJlcSA9IChjb3VudHMgLyBzdW0oY291bnRzKSkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoIC4sIGFlcyh4PSIiLCB5PWZyZXEsIGZpbGw9QVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5Iiwgd2lkdGg9MSwgY29sb3I9IndoaXRlIikgKwogIGNvb3JkX3BvbGFyKCJ5Iiwgc3RhcnQ9MTgwKSArCiAgdGhlbWVfdm9pZCgpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcGVyY2VudGFnZSksIHBvc2l0aW9uID0gcG9zaXRpb25fc3RhY2sodmp1c3QgPSAwLjUpLCBzaXplID0gMykgKwogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiU3BlY3RyYWwiKSArCiAgZ3VpZGVzKGZpbGwgPSBndWlkZV9sZWdlbmQodGl0bGUgPSAiQXNzZXNzbWVudCB0eXBlIikpICsKICBnZ3RpdGxlKCJQZXJjZW50YWdlIG9mIHN0dWRlbnRzIGluIGVhY2ggYXNzZXNzbWVudCB0eXBlIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSAKYGBgCgpgYGB7cn0KI2NhdGVnb3Jpc2UgaW50byBleGFtIGFuZCBjdyBvbmx5LiAKI1NvLSBleGFtID0gRXhhbSwgSW4tRGVwdCBFeGFtLCBjdyA9IENXLCBJbi1DbGFzcyBUZXN0IGFuZCBMYWIgVGVzdAoKY2xlYW5fbWFya19kYXRhW2NsZWFuX21hcmtfZGF0YSA9PSAiSW4tRGVwdCBFeGFtIl0gPC0gIkV4YW0iCmNsZWFuX21hcmtfZGF0YVtjbGVhbl9tYXJrX2RhdGEgPT0gIkluLUNsYXNzIFRlc3QiXSA8LSAiQ291cnNld29yayIKY2xlYW5fbWFya19kYXRhW2NsZWFuX21hcmtfZGF0YSA9PSAiTGFib3JhdG9yeSBUZXN0Il0gPC0gIkNvdXJzZXdvcmsiCgojZHJvcHBpbmcgbGV2ZWxzIAojaW4gZGVwdCBleGFtLCBpbiBjbGFzcyB0ZXN0IGFuZCBsYWIgdGVzdApkcm9wbGV2ZWxzKGNsZWFuX21hcmtfZGF0YSRBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpIC0+IGNsZWFuX21hcmtfZGF0YSRBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MKCnN1bW1hcnkoY2xlYW5fbWFya19kYXRhJEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykKYGBgCgpgYGB7cn0KI3Zpc3VhbGlzaW5nIHBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgdW5kZXIgZWFjaCBhc3Nlc3NtZW50IHR5cGUgYWZ0ZXIgY2F0ZWdvcmlzYXRpb24KY2xlYW5fbWFya19kYXRhICU+JQogIGdyb3VwX2J5KEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgc3VtbWFyaXNlKGNvdW50cyA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSAoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgZ2dwbG90KCAuLCBhZXMoeD0iIiwgeT1mcmVxLCBmaWxsPUFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykpICsKICBnZW9tX2JhcihzdGF0PSJpZGVudGl0eSIsIHdpZHRoPTEsIGNvbG9yPSJ3aGl0ZSIpICsKICBjb29yZF9wb2xhcigieSIsIHN0YXJ0PTApICsKICB0aGVtZV92b2lkKCkgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb24gPSBwb3NpdGlvbl9zdGFjayh2anVzdCA9IDAuNSksIHNpemUgPSAzKSArCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJTcGVjdHJhbCIpICsKICBndWlkZXMoZmlsbCA9IGd1aWRlX2xlZ2VuZCh0aXRsZSA9ICJBc3Nlc3NtZW50IHR5cGUiKSkgKwogIGdndGl0bGUoIlBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgaW4gZWFjaCBhc3Nlc3NtZW50IHR5cGUiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpIApgYGAKCmBgYHtyfQojY2xlYW5pbmcgcHJvZ3JhbW1lIHBhcnQKc3VtbWFyeShjbGVhbl9tYXJrX2RhdGEkUFJPR1JBTU1FX1BBUlQpCgojdmlzdWFsaXNlIG51bWJlciBvZiBzdHVkZW50cyBpbiBlYWNoIHByb2dyYW1tZSBwYXJ0CmdncGxvdChjbGVhbl9tYXJrX2RhdGEsIGFlcyh4ID0gUFJPR1JBTU1FX1BBUlQpKSArIAogIGdlb21fYmFyKGZpbGwgPSAiYmx1ZSIpICsKdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA2MCwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaGp1c3QgPSAxKSkgKwogIGdndGl0bGUoIk51bWJlciBvZiBzdHVkZW50cyBpbiBlYWNoIHBhcnQiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdQcm9ncmFtbWUgUGFydCcpICsKICB5bGFiKCdOdW1iZXIgb2Ygc3R1ZGVudHMnKSAgCmBgYApBcyBjYW4gYmUgc2VlbiBmcm9tIHRoZSBhYm92ZSB0YWJsZS0gdGhlcmUgYXJlIG1hbnkgcGFydHMtIGJ1dCBpbiB0aGlzIHN0dWR5IGkgYW0gb25seSBpbnRlcmVzdGVkIGluIFVHIGFuZCBQRyBzdHVkZW50cywgc28gd2lsbCByZW1vdmUgYWxsIGNhc2VzIHdoZXJlIG5vdCBBLCBCLCBDLCBELCBGLCBhbmQgVC4gCmBgYHtyfQojVGhlcmUgYXJlIDE2IHBhcnRzLSB3YW50IHRvIGtlZXAgNiBhbmQgcmVtb3ZlIDEwCgpzdWJzZXQoY2xlYW5fbWFya19kYXRhLCBQUk9HUkFNTUVfUEFSVCAhPSAiRyIgJiBQUk9HUkFNTUVfUEFSVCAhPSAiSSIgJiBQUk9HUkFNTUVfUEFSVCAhPSAiUiIgJiBQUk9HUkFNTUVfUEFSVCAhPSAiUjAiICYgUFJPR1JBTU1FX1BBUlQgIT0gIlIxIiAmIFBST0dSQU1NRV9QQVJUICE9ICJSMiIgJiBQUk9HUkFNTUVfUEFSVCAhPSAiUlQiICYgUFJPR1JBTU1FX1BBUlQgIT0gIlRPIiAmIFBST0dSQU1NRV9QQVJUICE9ICJUUiIgJiBQUk9HUkFNTUVfUEFSVCAhPSAiVFgiKSAtPiBjbGVhbl9tYXJrX2RhdGEKCmBgYAoKYGBge3J9CiNkcm9wcGluZyBsZXZlbHMgd2l0aCAwIApzdW1tYXJ5KGNsZWFuX21hcmtfZGF0YSRQUk9HUkFNTUVfUEFSVCkKCmRyb3BsZXZlbHMoY2xlYW5fbWFya19kYXRhJFBST0dSQU1NRV9QQVJUKSAtPiBjbGVhbl9tYXJrX2RhdGEkUFJPR1JBTU1FX1BBUlQKCnN1bW1hcnkoY2xlYW5fbWFya19kYXRhJFBST0dSQU1NRV9QQVJUKQpgYGAKCmBgYHtyfQojdmlzdWFsaXNlIG51bWJlciBvZiBzdHVkZW50cyBpbiBlYWNoIHByb2dyYW1tZSBwYXJ0cyB3ZSBhcmUgaW50ZXJlc3RlZCBpbgpnZ3Bsb3QoY2xlYW5fbWFya19kYXRhLCBhZXMoeCA9IFBST0dSQU1NRV9QQVJUKSkgKyAKICBnZW9tX2JhcihmaWxsID0gImJsdWUiKSArCnRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNjAsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGhqdXN0ID0gMSkpICsKICAgZ2d0aXRsZSgiTnVtYmVyIG9mIHN0dWRlbnRzIGluIGVhY2ggcGFydCIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ1Byb2dyYW1tZSBQYXJ0JykgKwogIHlsYWIoJ051bWJlciBvZiBzdHVkZW50cycpIApgYGAKCmBgYHtyfQojZGlzdHJpYnV0aW9uIG9mIG1hcmtzIGluIGNsZWFuZWQgbWFyayBkYXRhc2V0Cmhpc3QoY2xlYW5fbWFya19kYXRhJE1BUkssIAogICAgIG1haW49Ikhpc3RvZ3JhbSBvZiBtYXJrICgyMDE3LTIwMjEpIiwKeGxhYj0iTWFyayIsCnhsaW09YygwLDEwMCksCmNvbD0iZGFya21hZ2VudGEiKQojc2VlbXMgY2xvc2UgdG8gbm9ybWFsIGRpc3RyaWJ1dGlvbgpgYGAKCmBgYHtyfQojZnJvbSBhYm92ZSBsb29rcyBsaWtlIHRoZXJlIGFyZSBtYXJrcyBiZWxvdyAwCmNsZWFuX21hcmtfZGF0YSAlPiUKICBncm91cF9ieShNQVJLKSAlPiUKICB0YWxseSAoKQojUkVNT1ZFIE1BUktTIEJFTE9XLCBPTkUgRU5UUlkgT0YgLTMuIApmaWx0ZXIoY2xlYW5fbWFya19kYXRhLCBNQVJLID4gMCkgLT4gY2xlYW5fbWFya19kYXRhCiNub3cgODcwMDIxIHJvd3MKYGBgCgojICoqSm9pbmluZyB0aGUgdHdvIGRhdGFzZXRzKioKYGBge3J9CiNKT0lOIEVUSE5JQyBEQVRBIEFORCBNQVJLIERBVEEKaW5uZXJfam9pbihjbGVhbl9ldGhuaWNpdHlfZGF0YSwgY2xlYW5fbWFya19kYXRhKSAtPiBmdWxsX2RhdGEKcHJpbnQoZGlzdGluY3QoZnVsbF9kYXRhKSkKYGBgCgojICoqUmVzZWFyY2ggUXVlc3Rpb24gMSoqCiMgKioxLglXaGF0IGFmZmVjdCBkb2VzIHRoZSBhc3Nlc3NtZW50IHR5cGUgaGF2ZSBvbiB0aGUgc3R1ZGVudCBwZXJmb3JtYW5jZSBhbmQgdGhlIEJBTUUgYXdhcmRpbmcgZ2FwPyoqCgpUaGlzIGlzIGEgbXVsdGktbGV2ZWxlZCBxdWVzdGlvbi4gSSB3aWxsIGJlZ2FuIGJ5IGxvb2tpbmcgYXQgdGhlIGVmZmVjdCBvZiBldGhuaWNpdHkgYW5kIGFzc2Vzc21lbnQgdHlwZSBzZXBhcmF0ZWx5IG9uIHN0dWRlbnQgcGVyZm9ybWFuY2UuCgojIyAxLjEgRWZmZWN0IG9mIEV0aG5pY2l0eSBvbiBzdHVkZW50IHBlcmZvcm1hbmNlCgpWYXJpYW5jZSBpbiBtYXJrcyBmb3Igd2hpdGUgdnMgb3RoZXIKYGBge3J9CmJveHBsb3QoTUFSS35FdGhuaWNpdHksIGRhdGEgPSBmdWxsX2RhdGEpCmBgYApsb29rcyBsaWtlIGEgbG90IG9mIGRhdGEgcG9pbnRzIGFyZSBvdXRsaWVycy0gYnV0IGRhdGEgaXMgb3ZlciA1IHllYXJzIGFuZCB3ZSBhcmUgbW9zdGx5IGludGVyZXN0ZWQgaW4gdG9wIGdyYWRlcyBzbyB3aWxsIGZpbHRlciB0aGVzZSBvdXQgbGF0ZXIuIAoKQW5hbHlzaXMgb2YgYXZlcmFnZSBNYXJrcyBvZiBXaGl0ZSB2cyBCQU1FL090aGVyIHN0dWRlbnRzICgyMDE3LTIwMjEpCgpCZWxvdyBpcyBiYXIgY2hhcnQgc2hvd2luZyBvZiBlZmZlY3Qgb2YgZXRobmljaXR5IG9uIG1lYW4gbWFyayBiZXR3ZWVuIDIwMTctMjAyMQoKYGBge3J9CmZ1bGxfZGF0YSAlPiUKICBzZWxlY3QoRXRobmljaXR5LCBNQVJLKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHkpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgZ2dwbG90KGFlcyh4PUV0aG5pY2l0eSwgeSA9IEF2ZXJhZ2VfbWFyaywgZmlsbCA9RXRobmljaXR5KSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYWlyZWQiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHJvdW5kKEF2ZXJhZ2VfbWFyaywgMikpLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjkpLCB2anVzdD0tMC4yNSkgKwogIGdndGl0bGUoIkF2ZXJhZ2UgVGVzdCBTY29yZXMgYnkgRXRobmljaXR5IGJldHdlZW4gMjAxNy0yMDIxIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeWxhYignQXZlcmFnZSBNYXJrJykgCgpgYGAKTmV4dCwgSSB3YW50ZWQgdG8gc2VlIHRoZSBhZmZlY3Qgb2YgZXRobmljaXR5IG9uIGF2ZXJhZ2UgbWFyayBlYWNoIHllYXIgYmV0d2VlbiAyMDE3LTIwMjEuClRoaXMgdGFibGUgYmVsb3cgc2hvd3MgdGhlIGdhcCBpbiBhdmVyYWdlIGdyYWRlIGluIHdoaXRlIHZzIEJhbWUgc3R1ZGVudHMgaXMgcmVkdWNpbmcgb3ZlciB0aGUgeWVhcnMuIFRoaXMgcmVkdWN0aW9uIGlzIHZlcnkgc21hbGwgY2hhbmdpbmcgYnkganVzdCAxLjQgb3ZlciA1IHllYXJzLiAKYGBge3J9CiNXaGl0ZSB2cyBub24gd2hpdGUgZ3JhZGVzIHRhYmxlIGxpa2UgaW4gc2NyZWVuc2hvdCBpbiByZXBvcnQKZnVsbF9kYXRhICU+JQogIHNlbGVjdChJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBFdGhuaWNpdHksTUFSSykgJT4lCiAgZ3JvdXBfYnkoSU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgRXRobmljaXR5KSAlPiUKICB0YWxseShtZWFuKE1BUkspKSAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgc3ByZWFkKGtleSA9IEV0aG5pY2l0eSwgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIG11dGF0ZShHYXAgPSBXaGl0ZSAtIEJBTUUpCmBgYAoKQmVsb3cgaXMgYSBsaW5lIGdyYXBoIHRvIHZpc3VhbGlzZSB0aGUgYWJvdmUgdGFibGUuIFRoaXMgaXMgc2hvd2luZyBnYXAgaXMgcmVkdWNpbmcsIGJ1dCBib3RoIHdoaXRlIGFuZCBiYW1lIHN0dWRlbnRzIGF2ZXJhZ2UgZ3JhZGUgc2VlbXMgdG8gYmUgcmVkdWNpbmcgb3ZlciB0aGUgeWVhcnMuIApgYGB7cn0KZnVsbF9kYXRhICU+JQogIHNlbGVjdChJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBFdGhuaWNpdHksTUFSSykgJT4lCiAgZ3JvdXBfYnkoSU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgRXRobmljaXR5KSAlPiUKICB0YWxseShtZWFuKE1BUkspKSAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgI3NwcmVhZChrZXkgPSBFdGhuaWNpdHksIHZhbHVlID0gQXZlcmFnZV9tYXJrKSAlPiUKICMgbXV0YXRlKEdhcCA9IFdoaXRlIC0gQkFNRSkgJT4lCiAgZ2dwbG90KGFlcyh4ID0gSU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgeSA9IEF2ZXJhZ2VfbWFyaywgY29sID0gRXRobmljaXR5KSkgKwogIGdlb21fbGluZSgpICsKICBnZW9tX3BvaW50KCkgKwogIGdndGl0bGUoIkF2ZXJhZ2UgZ3JhZGUgZm9yIHdoaXRlIHZzIEJBTUUgc3R1ZGVudHMgKDIwMTctMjAyMSkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBY2FkZW1pYyBZZWFyJykgKwogIHlsYWIoJ0F2ZXJhZ2UgTWFyaycpICsKICBsYWJzKGNvbD0nRXRobmljaXR5JykgCmBgYAojIyMgQW5hbHlzaXMgb2YgQkFNRSBBd2FyZGluZyBHYXAgKDIwMTctMjAyMSkKVGhlIGJlbG93IGJhciBncmFwaCBpcyBzaG93aW5nIHBlcmNlbnRhZ2Ugb2Ygd2hpdGUgdnMgQkFNRSBzdHVkZW50cyBhY2hpZXZpbmcgdG9wIGRlZ3JlZSAoMjAxNy0yMDIxKQpOT1RFLSBmb3IgbWV0aG9kb2xvZ3kvcmVzdWx0cwpJIGdyb3VwZWQgYnkgSUQyIGZpcnN0IGluIG9yZGVyIHRvIGNhbGN1bGF0ZSBhdmVyYWdlIGdyYWRlIGZvciBlYWNoIHN0dWRlbnQuLnRoZW4gaSBmaWx0ZXJlZCBmb3Igc3R1ZGVudHMgd2hvIGdvdCBhbiBhdmVyYWdlIGdyYWRlIGFib3ZlIDYwClRoZW4gZ3JvdXBlZCBieSBldGhuaWNpdHkKVGhpcyBpcyBpbXBvcnRhbnQgYXMgdGhlIGRhdGEgaXMgbm90IGltcG9ydGFudCwgaWYgbm90IGdyb3VwZWQgYnkgSUQyIHJlc3VsdHMgd291bGQgc2hvdyBhbnkgc3R1ZGVudCB3aG8gZ290IGFib3ZlIDYwIGZvciBhdCBsZWFzdCAxIGFzc2Vzc21lbnQKCmBgYHtyfQpmdWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoSUQyKSAlPiUKICBtdXRhdGUoQXZlcmFnZV9tYXJrID0gbWVhbihNQVJLKSkgJT4lCiAgZmlsdGVyKEF2ZXJhZ2VfbWFyayA+PTYwKSAlPiUKICBzZWxlY3QoSUQyLCBFdGhuaWNpdHksIEF2ZXJhZ2VfbWFyaykgJT4lCiAgZGlzdGluY3QoKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHkpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9RXRobmljaXR5LCB5ID0gY291bnRzLCBmaWxsID1FdGhuaWNpdHkpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIHBvc2l0aW9uID0gJ2RvZGdlJykgKyAKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlBhc3RlbDEiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHBlcmNlbnRhZ2UpLCBwb3NpdGlvbiA9IHBvc2l0aW9uX3N0YWNrKHZqdXN0ID0gMS4xKSwgc2l6ZSA9IDMpICArCiAgZ2d0aXRsZSgiUGVyY2VudGFnZSBvZiB3aGl0ZSB2cyBCQU1FIHN0dWRlbnRzIGFjaGlldmluZyB0b3AgZGVncmVlICgyMDE3LTIwMjEpIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeWxhYignTnVtYmVyIG9mIHN0dWRlbnRzJykKYGBgCgpCZWxvdyBncmFwaCBzaG93cyBkaWZmZXJlbmNlIGJldHdlZW4gbnVtYmVyIG9mIFdoaXRlIHZzIEJBTUUgc3R1ZGVudHMgYWNoaWV2aW5nIHRvcCBkZWdyZWUgKEJBTUUgYXdhcmRpbmcgZ2FwKSBpcyBjb25zaXN0ZW50IG92ZXIgeWVhcnMgMjAxNy0yMDIxCmBgYHtyfQojVmlzdWFsaXNpbmcgcGVyY2VudGFnZSBvZiB3aGl0ZSB2cyBCQU1FIHN0dWRlbnRzIGFjaGlldmluZyB0b3AgZGVncmVlIGVhY2ggeWVhciAoMjAxNy0yMDIxKQpmdWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoSUQyKSAlPiUKICBtdXRhdGUoQXZlcmFnZV9tYXJrID0gbWVhbihNQVJLKSkgJT4lCiAgZmlsdGVyKEF2ZXJhZ2VfbWFyayA+PTYwKSAlPiUKICBzZWxlY3QoSUQyLCBFdGhuaWNpdHksIEF2ZXJhZ2VfbWFyaywgSU5TVEFOQ0VfQUNBREVNSUNfWUVBUikgJT4lCiAgZGlzdGluY3QoKSAlPiUKICBncm91cF9ieShJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBFdGhuaWNpdHkpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9SU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgeSA9IGNvdW50cywgZmlsbCA9RXRobmljaXR5KSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYXN0ZWwxIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9MC45KSwgdmp1c3Q9LTAuMjUpICsKICBnZ3RpdGxlKCJQZXJjZW50YWdlIG9mIHdoaXRlIHZzIEJBTUUgc3R1ZGVudHMgYWNoaWV2aW5nIHRvcCBkZWdyZWUgZWFjaCB5ZWFyICgyMDE3LTIwMjEpIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeGxhYignQWNhZGVtaWMgWWVhcicpICsKICB5bGFiKCdOdW1iZXIgb2Ygc3R1ZGVudHMnKQpgYGAKUHJvYmxlbSB3aXRoIGFib3ZlIGdyYXBocyBpcyB0aGF0IHRoZXJlIGFyZSBhIGxvdCBtb3JlIHdoaXRlIHN0dWRlbnRzIGluIHRoZSB1bml2ZXJzaXR5LCBhbmQgdGhlcmVmb3JlIGl0IGlzIGEgZ2l2ZW4gdGhhdCBtb3JlIHdoaXRlIHN0dWRlbnRzIGFyZSBhY2hpZXZpbmcgdG9wIGRlZ3JlZXMuIApTbyBpbnN0ZWFkLCB0aGlzIGdyYXBoIGNsZWFybHkgc2hvd3MgZGlzdHJpYnV0aW9uIG9mIGRlZ3JlZSBvdXRjb21lcyBpbnN0ZWFkLlRoZSBmb2xsb3dpbmcgZ3JhcGggbWFrZXMgdGhlIHJlc3VsdHMgY2xlYXJlciBmb3IgaW50ZXJwcmV0YXRpb24vY29tcGFyaXNvbnMuCkZvciBleGFtcGxlIGhlcmUsIDM3JSBvZiB3aGl0ZSBzdHVkZW50cyBnZXQgYSBmaXJzdCBpbiBjb21wYXJpc29uIHRvIG9ubHkgMjEuOSUgb2YgQkFNRSBzdHVkZW50cy4KCmBgYHtyfQojdmlzdWFsaXNhdGlvbiBvZiBkaXN0cmlidXRpb24gb2YgZGVncmVlIG91dGNvbWVzCmZ1bGxfZGF0YSAlPiUKICBncm91cF9ieShJRDIpICU+JQogIG11dGF0ZShBdmVyYWdlX21hcmsgPSBtZWFuKE1BUkspKSAlPiUKICBtdXRhdGUoR3JhZGUgPSBjYXNlX3doZW4oQXZlcmFnZV9tYXJrID49IDcwIH4gIkZpcnN0IENsYXNzIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgQXZlcmFnZV9tYXJrID49IDYwIH4gIlVwcGVyIFNlY29uZCBDbGFzcyBIb25vdXIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICBBdmVyYWdlX21hcmsgPj0gNTAgfiAiTG93ZXIgU2Vjb25kIENsYXNzIEhvbm91ciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIEF2ZXJhZ2VfbWFyayA+PSA0MCB+ICJUaGlyZCBjbGFzcy9wYXNzIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgQXZlcmFnZV9tYXJrIDwgNDAgfiAiRmFpbCIpKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHksIEdyYWRlKSAlPiUKICBzdW1tYXJpc2UoY291bnRzID0gbigpKSAlPiUKICBtdXRhdGUoZnJlcSA9IChjb3VudHMgLyBzdW0oY291bnRzKSkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9RXRobmljaXR5LCB5ID0gZnJlcSwgZmlsbCA9R3JhZGUpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScpICsgCiAgY29vcmRfZmxpcCgpICsKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlBhc3RlbDEiKSArCiAgZ2VvbV90ZXh0X3JlcGVsKGFlcyhsYWJlbCA9IHBlcmNlbnRhZ2UpLCBwb3NpdGlvbiA9IHBvc2l0aW9uX3N0YWNrKHZqdXN0ID0gMC41KSwgc2l6ZSA9IDIuNSkgKwogIGdndGl0bGUoIkRpc3RyaWJ1dGlvbiBvZiBkZWdyZWUgb3V0Y29tZXMgYnkgZXRobmljIGdyb3VwICgyMDE3LTIwMjEpIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgbGFicyh5ID0gIkZyZXF1ZW5jeSIsIGZpbGwgPSAiRGVncmVlIENsYXNzaWZpY2F0aW9uIikKYGBgCgojIyAxLjIgRWZmZWN0IG9mIGFzc2Vzc21lbnQgdHlwZSBvbiBzdHVkZW50IHBlcmZvcm1hbmNlCgpJIGhhdmUgYmVnYW4gYnkgdmlzdWFsaXNpbmcgdGhlIGF2ZXJhZ2UgbWFyayBmb3IgZWFjaCBhc3Nlc3NtZW50IHR5cGUgKDIwMTctMjAyMSkKYGBge3J9CiNWaXN1YWxpc2F0aW9uIG9mIGVmZmVjdCBvZiBhc3Nlc3NtZW50IHR5cGUgb24gYXZlcmFnZSBtYXJrIGJldHdlZW4gMjAxNy0yMDIxCmZ1bGxfZGF0YSAlPiUKICBzZWxlY3QoQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCBNQVJLKSAlPiUKICBncm91cF9ieShBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgICNzcHJlYWQoa2V5ID0gRXRobmljaXR5LCB2YWx1ZSA9IEF2ZXJhZ2VfbWFyaykgJT4lCiAgZ2dwbG90KGFlcyh4PUFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgeSA9IEF2ZXJhZ2VfbWFyaywgZmlsbCA9IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykpICsKICBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JywgcG9zaXRpb24gPSAnZG9kZ2UnKSArIAogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiUGFzdGVsMiIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcm91bmQoQXZlcmFnZV9tYXJrLCAyKSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiQXZlcmFnZSBNYXJrIGZvciBDb3Vyc2V3b3JrIHZzIEV4YW0gKDIwMTctMjAyMSkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBc3Nlc3NtZW50IFR5cGUnKSArCiAgeWxhYignQXZlcmFnZSBNYXJrJykgKwogIGd1aWRlcyhmaWxsID0gZ3VpZGVfbGVnZW5kKHRpdGxlID0gIkFzc2Vzc21lbnQgdHlwZSIpKSAKYGBgClRoaXMgYmFyIGdyYXBoIGFib3ZlIGlzIHNob3dpbmcgdGhhdCBzdHVkZW50cyBvdmVyYWxsIHBlcmZvcm0gbXVjaCBiZXR0ZXIgaW4gY291cnNld29yayB0aGFuIGV4YW1zLi4uYnV0IGRvZXMgZXRobmljaXR5IGhhdmUgYW4gZWZmZWN0IG9uIHRoaXM/PyBXaWxsIGxvb2sgYXQgdGhpcyBsYXRlcgoKTmV4dCBpIHdhbnRlZCB0byBldmFsdWF0ZSB0aGUgYXZlcmFnZSB5ZWFybHkgbWFyayBkZXBlbmRlbnQgb24gYXNzZXNzbWVudCB0eXBlLiBUaGlzIHNob3dzIGV2ZXJ5IHllYXIgdGhlIGF2ZXJhZ2UgZ3JhZGUgaXMgbXVjaCBiaWdnZXIgZm9yIGN3IHR5cGUgYXNzZXNzbWVudHMgdnMgZXhhbXMuIEVzc2VudGlhbGx5IHN0dWRlbnRzIHBlcmZvcm0gbXVjaCBiZXR0ZXIgaW4gY291cnNld29yayB0eXBlIGFzc2Vzc21lbnRzIGluIGNvbXBhcmlzb24gdG8gZXhhbXMuIApgYGB7cn0KZnVsbF9kYXRhICU+JQogIHNlbGVjdChJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsTUFSSykgJT4lCiAgZ3JvdXBfYnkoSU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICB0YWxseShtZWFuKE1BUkspKSAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgc3ByZWFkKGtleSA9IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIG11dGF0ZShHYXAgPSBDb3Vyc2V3b3JrIC0gRXhhbSkKYGBgCgpUaGlzIGxpbmUgZ3JhcGggYmVsb3cgc2hvd3MgbWFueSB0aGluZ3M6CjEuIHN0dWRlbnRzIHBlcmZvcm0gbXVjaCBiZXR0ZXIgaW4gY3cgdnMgZXhhbXMKMi4gQXZlcmFnZSBjdyBtYXJrIGlzIHJlZHVjaW5nIG92ZXIgdGhlIHllYXJzCjMuIEF2ZXJhZ2UgZXhhbSBtYXJrIGlzIGluY3JlYXNpbmcgb3ZlciB0aGUgeWVhcnMsIGJ1dCBnb2VzIGRvd24gaW4gMjAyMQo0LiBhc3Nlc3NtZW50IGF3YXJkaW5nIGdhcCBpcyByZWR1Y2luZy4gCmBgYHtyfQojVmlzdWFsaXNhdGlvbiBvZiBhYm92ZQpmdWxsX2RhdGEgJT4lCiAgc2VsZWN0KElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyxNQVJLKSAlPiUKICBncm91cF9ieShJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICAjc3ByZWFkKGtleSA9IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogICNtdXRhdGUoR2FwID0gQ291cnNld29yayAtIEV4YW0pICU+JQogIGdncGxvdChhZXMoeCA9IElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIHkgPSBBdmVyYWdlX21hcmssIGNvbCA9IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykpICsKICBnZW9tX2xpbmUoKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZ3RpdGxlKCJBdmVyYWdlIHllYXJseSBNYXJrIGZvciBDb3Vyc2V3b3JrIHZzIEV4YW0gKDIwMTctMjAyMSkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBY2FkZW1pYyBZZWFyJykgKwogIHlsYWIoJ0F2ZXJhZ2UgTWFyaycpICsKICBsYWJzKGNvbD0nQXNzZXNzbWVudCBUeXBlJykgCgpgYGAKCkFuYWx5c2lzIG9mIGVmZmVjdCBvZiBhc3Nlc3NtZW50IHR5cGUgb24gYWNoaWV2aW5nIHRvcCBkZWdyZWUoMjAxNy0yMDIyKQoKV2hhdCBhYm91dCBlZmZlY3Qgb2YgYXNzZXNzbWVudCB0eXBlIG9uIHRvcCBkZWdyZWVzPwpUaGlzIGJhciBjaGFydCBzaG93cyB0aGUgYXZlcmFnZSBudW1iZXIgb2Ygc3R1ZGVudHMgdGhhdCBkaWQgY3cgdnMgZXhhbSBpbiBhbGwgc3R1ZGVudHMgdGhhdCBhY2hpZXZlZCBhIHRvcCBkZWdyZWUuIAoKYGBge3J9CiNhdmVyYWdlIHBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgd2hvIGRpZCBjdyB2cyBleGFtIHRvIGdldCBmaXJzdApmdWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoSUQyKSAlPiUKICBtdXRhdGUoQXZlcmFnZV9tYXJrID0gbWVhbihNQVJLKSkgJT4lCiAgZmlsdGVyKEF2ZXJhZ2VfbWFyayA+PTYwKSAlPiUKICBzZWxlY3QoSUQyLCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIEF2ZXJhZ2VfbWFyaykgJT4lCiAgZGlzdGluY3QoKSAlPiUKICBncm91cF9ieShBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9QVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCB5ID0gY291bnRzLCBmaWxsID0gQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYXN0ZWwyIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9MC45KSwgdmp1c3Q9LTAuMjUpICsKICBnZ3RpdGxlKCJQZXJjZW50YWdlIG9mIHN0dWRlbnRzIGRvaW5nIGN3IHZzIGV4YW0gZm9yIHRvcCBkZWdyZWUiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBc3Nlc3NtZW50IFR5cGUnKSArCiAgeWxhYignTnVtYmVyIG9mIHN0dWRlbnRzJykgKwogIGd1aWRlcyhmaWxsID0gZ3VpZGVfbGVnZW5kKHRpdGxlID0gIkFzc2Vzc21lbnQgdHlwZSIpKSAKYGBgCgpUaGlzIGJhciBjaGFydCBpcyBzaG93aW5nIHllYXJseSBhdmVyYWdlIHBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgd2hvIGRpZCBjdyB2cyBleGFtIHRvIGdldCBmaXJzdApUaGlzIHJlc3VsdHMgYXJlIHByZXR0eSBjb25zaXN0ZW50IG92ZXIgdGhlIHllYXJzLiBIYXMgcmVkdWNlZCBzbGlnaHRseSBvdmVyIHRoZSB5ZWFycy4KR2FwIGluIDIxIGxvb2tzIHByZXR0eSBzbWFsbCwgYnV0IG51bWJlciBvZiBzdHVkZW50cyBpbiB0aGlzIHllYXIgd2FzIGxvd2VyLiAKCmBgYHtyfQojcGVyY2VudGFnZSBvZiBzdHVkZW50cyB3aG8gZGlkIGN3IHZzIGV4YW0gdG8gZ2V0IGZpcnN0ICgyMDE3LTIwMjIpCiAgZnVsbF9kYXRhICU+JQogIGdyb3VwX2J5KElEMikgJT4lCiAgbXV0YXRlKEF2ZXJhZ2VfbWFyayA9IG1lYW4oTUFSSykpICU+JQogIGZpbHRlcihBdmVyYWdlX21hcmsgPj02MCkgJT4lCiAgc2VsZWN0KElEMiwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCBBdmVyYWdlX21hcmssIElOU1RBTkNFX0FDQURFTUlDX1lFQVIpICU+JQogIGRpc3RpbmN0KCkgJT4lCiAgZ3JvdXBfYnkoSU5TVEFOQ0VfQUNBREVNSUNfWUVBUiwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBzdW1tYXJpc2UoY291bnRzID0gbigpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgZ2dwbG90KGFlcyh4PUlOU1RBTkNFX0FDQURFTUlDX1lFQVIsIHkgPSBjb3VudHMsIGZpbGwgPUFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykpICsKICBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JywgcG9zaXRpb24gPSAnZG9kZ2UnKSArIAogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiUGFzdGVsMSIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcGVyY2VudGFnZSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiUGVyY2VudGFnZSBvZiBzdHVkZW50cyBkb2luZyBjdyB2cyBleGFtIGZvciB0b3AgZGVncmVlIGVhY2ggeWVhciIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ0FjYWRlbWljIFllYXInKSArCiAgeWxhYignTnVtYmVyIG9mIHN0dWRlbnRzJykgKwogIGd1aWRlcyhmaWxsID0gZ3VpZGVfbGVnZW5kKHRpdGxlID0gIkFzc2Vzc21lbnQgdHlwZSIpKSAKYGBgCgojIyAxLjMgQW5hbHlzaW5nIGVmZmVjdCBvZiBhc3Nlc3NtZW50IHR5cGUgb24gc3R1ZGVudCBwZXJmb3JtYW5jZSBhbmQgQkFNRSBhd2FyZGluZyBnYXAKCmBgYHtyfQojQ2hlY2tpbmcgdGhlcmUgaXMgYSByZWFzb25hYmxlIG51bWJlciBvZiB3aGl0ZS9vdGhlcnMgaW4gZWFjaCBhc3Nlc3NtZW50IHR5cGUuIApmdWxsX2RhdGEgJT4lCiAgc2VsZWN0KEV0aG5pY2l0eSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgdGFsbHkoKQpgYGAKRmlyc3QgaSB3YW50IHRvIGxvb2sgYXQgYWZmZWN0IG9mIGFzc2Vzc21lbnQgdHlwZSBvbiBzdHVkZW50IHBlcmZvcm1hbmNlKFdoaXRlIHZzIE90aGVyKQpUaGlzIHRhYmxlIGJlbG93IHNob3dzIHRoZSBkaWZmZXJlbmNlIGluIGF2Z2VyYWdlIGN3IGFuZCBleGFtIG1hcmtzIGZvciB3aGl0ZSBWcyBCYW1lIHN0dWRlbnRzIAoKQXZlcmFnZSBncmFkZSBvdmVyIHBhc3QgNSB5ZWFycyBmb3IgYXNzZXNzbWVudCB0eXBlIChBVCkgYW5kIGV0aG5pY2l0eQpgYGB7cn0KZnVsbF9kYXRhICU+JQogIHNlbGVjdChFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyxNQVJLKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgdGFsbHkobWVhbihNQVJLKSkgICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICAgc3ByZWFkKGtleSA9IEV0aG5pY2l0eSwgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIG11dGF0ZShnYXAgPSBXaGl0ZSAtIEJBTUUpCmBgYApUaGUgZ2FwIGRvZXNudCBsb29rIGh1Z2UsIGJ1dCBpdCBpcyBiaWdnZXIgZm9yIGNvdXJzZXdvcmsgdHlwZSBhc3Nlc3NtZW50cy4gSG93ZXZlciB0aGlzIGlzIGdhcCBpbiBhdmVyYWdlIG1hcmsgKHN0dWRlbnQgcGVyZm9ybWFuY2UpLCBub3QgdGhlIG51bWJlciBvZiBzdHVkZW50cyB3aG8gYWNoaWV2ZWQgYSB0b3AgZGVncmVlLiBUaGVyZWZvcmUsIHRoaXMgaXMgbm90IHJlbGF0aW5nIHRvIEJhbWUgYXdhcmRpbmcgZ2FwLiBTdGlsbCBpbnRlcmVzdGluZy4uLgoKVmlzdWFsaXNhdGlvbiBvZiBhYm92ZSB0YWJsZQpMb29raW5nIGF0IGF2ZXJhZ2UgZ3JhZGUgb3ZlciBwYXN0IDUgeWVhcnMgZm9yIEFUIGFuZCBldGhuaWNpdHkKYGBge3J9CmZ1bGxfZGF0YSAlPiUKICBzZWxlY3QoRXRobmljaXR5LCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsTUFSSykgJT4lCiAgZ3JvdXBfYnkoRXRobmljaXR5LCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgICNzcHJlYWQoa2V5ID0gRXRobmljaXR5LCB2YWx1ZSA9IEF2ZXJhZ2VfbWFyaykgJT4lCiAgZ2dwbG90KGFlcyh4PUFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgeSA9IEF2ZXJhZ2VfbWFyaywgZmlsbCA9IEV0aG5pY2l0eSkpICsKICBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JywgcG9zaXRpb24gPSAnZG9kZ2UnKSAgKyAKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIkFjY2VudCIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcm91bmQoQXZlcmFnZV9tYXJrLCAyKSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiQXZlcmFnZSBNYXJrIEJ5IEFzc2Vzc21lbnQgVHlwZSBBbmQgRXRobmljaXR5IikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeGxhYignQXNzZXNzbWVudCBUeXBlJykgCiAgeWxhYignQXZlcmFnZSBNYXJrJykKCmBgYApUaGlzIGdyYXBoIGFib3ZlIGlzIHVzZWZ1bCBhcyBpdCBzaG93cyBkaWZmZXJlbmNlIGluIGF2ZXJhZ2UgbWFyayBmb3IgZWFjaCBhc3Nlc3NtZW50IHR5cGUgYnkgZXRobmljaXR5LgpIb3dldmVyIHRoaXMgcmVzZWFyY2ggcXVlc3Rpb24gcGVydGFpbnMgdG8gdGhlIGFmZmVjdCBvZiBhc3Nlc3NtZW50IHR5cGUgb24gQkFNRSBhd2FyZGluZyBnYXBzLgpUaGVyZWZvcmUsIGluc3RlYWQgb2YgYXZlcmFnZSBtYXJrcywgd2UgbmVlZCB0byBjb21wYXJlIHRoZSBudW1iZXIgb2Ygd2hpdGUgdnMgQkFNRSBzdHVkZW50cyB3aG8gYWNoaWV2ZWQgdG9wIGRlZ3JlZXMgYW5kIHNlZSBpZiBnYXAgaXMgZ3JlYXRlciBmb3IgY3cgdHlwZSBhc3Nlc3NtZW50cyBvciBjb3Vyc2V3b3JrLiAKCmBgYHtyfQpmdWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoSUQyKSAlPiUKICBtdXRhdGUoQXZlcmFnZV9tYXJrID0gbWVhbihNQVJLKSkgJT4lCiAgZmlsdGVyKEF2ZXJhZ2VfbWFyayA+PTYwKSAlPiUKICBzZWxlY3QoSUQyLCBFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgQXZlcmFnZV9tYXJrKSAlPiUKICBkaXN0aW5jdCgpICU+JQogIHNlbGVjdChFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgZ3JvdXBfYnkoRXRobmljaXR5LEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgc3VtbWFyaXNlKGNvdW50cyA9IG4oKSkgJT4lCiAgbXV0YXRlKHBlcmNlbnRhZ2UgPSBzY2FsZXM6OmxhYmVsX3BlcmNlbnQoKShjb3VudHMgLyBzdW0oY291bnRzKSkpICU+JQogICNzZWxlY3QoRXRobmljaXR5LCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIHBlcmNlbnRhZ2UpICU+JQogIGdncGxvdChhZXMoeD1BU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIHkgPSBwZXJjZW50YWdlLCBmaWxsID0gRXRobmljaXR5KSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICArIAogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiU2V0MSIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcGVyY2VudGFnZSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiRWZmZWN0IG9mIGFzc2Vzc21lbnQgdHlwZSBvbiBhd2FyZGluZyBnYXAiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBc3Nlc3NtZW50IFR5cGUnKSArCiAgeWxhYignUGVyY2VudGFnZSBvZiBzdHVkZW50cycpCmBgYApUaGUgcmVzdWx0cyBoZXJlIGFyZSBpbnRlcmVzdGluZy4gVGhlcmUgaXMgYSA0JSBhd2FyZGluZyBnYXAgZm9yIGJvdGggZXRobmljaXRpZXMgaW4gZWFjaCBhc3Nlc3NtZW50IHR5cGUtIHRoaXMgc3VnZ2VzdHMgZXhhbSBkbyBub3QgbGVhZCB0byBhIHdpZGVyIGF3YXJkaW5nIGdhcCBmb3IgdGhlIHllYXJzIDIwMTctMjAyMS5XaGl0ZSBzdHVkZW50cyBzdGlsbCBwZXJmb3JtaW5nIGJldHRlciBmb3IgYm90aCBhc3Nlc3NtZW50IHR5cGVzLCBidXQgYm90aCBldGhuaWNpdGllcyBwZXJmb3JtaW5nIHNpbWlsYXIgZm9yIGFzc2Vzc21lbnQgdHlwZS4gCgpgYGB7cn0KI2ludGVyYWN0aW9uIHBsb3Qgb2YgZnVsbCBkYXRhc2V0LiAKaW50ZXJhY3Rpb24ucGxvdCh4LmZhY3RvciA9IGZ1bGxfZGF0YSRFdGhuaWNpdHksICN4LWF4aXMgdmFyaWFibGUKICAgICAgICAgICAgICAgICB0cmFjZS5mYWN0b3IgPSBmdWxsX2RhdGEkQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCAjdmFyaWFibGUgZm9yIGxpbmVzCiAgICAgICAgICAgICAgICAgcmVzcG9uc2UgPSBmdWxsX2RhdGEkTUFSSywgI3ktYXhpcyB2YXJpYWJsZQogICAgICAgICAgICAgICAgIGZ1biA9IG1lZGlhbiwgI21ldHJpYyB0byBwbG90CiAgICAgICAgICAgICAgICAgeWxhYiA9ICJBdmVyYWdlIE1hcmsiLAogICAgICAgICAgICAgICAgIHhsYWIgPSAiRXRobmljaXR5IiwKICAgICAgICAgICAgICAgICBjb2wgPSBjKCJyZWQiLCAiYmx1ZSIpLAogICAgICAgICAgICAgICAgIGx0eSA9IDEsICNsaW5lIHR5cGUKICAgICAgICAgICAgICAgICBsd2QgPSAyLCAjbGluZSB3aWR0aAogICAgICAgICAgICAgICAgIHRyYWNlLmxhYmVsID0gIkFzc2Vzc21lbnQgdHlwZSIpCmBgYAoKIyBTdGF0aXN0aWNhbCBBbmFseXNpcy9Nb2RlbGxpbmcKCiMjIE1peGVkIG1vZGVsIDEKCmBgYHtyfQojUXVlc3Rpb24gMToKIwkxLglXaGF0IGFmZmVjdCBkb2VzIHRoZSBhc3Nlc3NtZW50IHR5cGUgaGF2ZSBvbiBzdHVkZW50IHBlcmZvcm1hbmNlPwoKI2ZpdHRpbmcgbW9kZWwKbW1vZGVsMSA8LSBsbWVyKE1BUksgfiBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MgKiBFdGhuaWNpdHkgKyBQUk9HUkFNTUVfUEFSVCArICgxfElEMikgKyAoMXxNT0RVTEVfQ09ERSksIGRhdGEgPSBmdWxsX2RhdGEpCmBgYAoKYGBge3J9CiNzaG93aW5nIG1vZGVsIHJlc3VsdHMKc3VtbWFyeShtbW9kZWwxKQpgYGAKCgpgYGB7cn0KI2ludGVyYWN0aW9uIHBsb3Qgb2YgbW9kZWwuLi4KCnBsb3RfbW9kZWwobW1vZGVsMSwgdHlwZSA9ICJwcmVkIiwgdGVybXMgPSBjKCJFdGhuaWNpdHkiLCAiQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDIikpCmBgYAoKYGBge3J9CmFub3ZhKG1tb2RlbDEpCmBgYApSZXN1bHRzIGZyb20gdGhlIGFub3ZhIHRlc3Qgc2hvdyBlYWNoIG9mIHRoZXNlIHZhcmlhYmxlcyBoYXZlIHNpZ25pZmljYW50IGVmZmVjdCBvbiBtYXJrCgpgYGB7cn0KI2VmZmVjdCBzaXplCmVmZmVjdHNpemU6OmV0YV9zcXVhcmVkKG1tb2RlbDEsIHBhcnRpYWwgPSBUUlVFKQpgYGAKCmBgYHtyfQojZW1tZWFucyBpcyBtZWFudCB0byBiZSBsb29rZWQgYXQgZm9yIG1haW4gZWZmZWN0LgojaW4gdGhpcyBjYXNlIHRoZXkgd2VyZSBhbGwgc2lnbmlmaWNhbnQgc28gd2lsbCBkbyBmb3IgZWFjaAoKZW1tZWFucyhtbW9kZWwxLCBsaXN0KHBhaXJ3aXNlfkFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyksIGFkanVzdD0idHVrZXkiKQoKI2hlcmUgY29tcGFyaXNvbiBzaG93cyB0aGF0IHRoZXJlIGlzIGEgc2lnbmlmaWNhbnQgZGlmZmVyZW5jZSBpbiBtYXJrIHdoZW4gYXNzZXNzbWVudCB0eXBlIGlzIGN3IGNvbXBhcmVkIHRvIGV4YW0gKHAudmFsdWUgPSA8MC4wMDAxKQpgYGAKCmBgYHtyfQojY29tcHV0aW5nIGVmZmVjdCBzaXplcyBmcm9tIGVtbWVhbi0gZmlyc3RseSBlZmZlY3Qgb2YgYXNzZXNzbWVudCB0eXBlCiNjb2RlIGFkYXB0ZWQgZnJvbSBodHRwczovL3NlYXJjaC5yLXByb2plY3Qub3JnL0NSQU4vcmVmbWFucy9lbW1lYW5zL2h0bWwvZWZmX3NpemUuaHRtbApWYXJDb3JyKG1tb2RlbDEpCnRvdFNEIDwtIHNxcnQoIDYuMjMxNSArIDUuMDQzMyArICAxMi4zODk4KQplbW1WIDwtIGVtbWVhbnMobW1vZGVsMSwgfiBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpCmVmZl9zaXplKGVtbVYsIHNpZ21hID0gdG90U0QsIGVkZiA9IDUpCmBgYAoKYGBge3J9CmVtbWVhbnMobW1vZGVsMSwgbGlzdChwYWlyd2lzZX5FdGhuaWNpdHkpLCBhZGp1c3Q9InR1a2V5IikKYGBgCgpgYGB7cn0KI2VmZmVjdCBzaXplIG9mIGV0aG5pY2l0eQplbW1WMiA8LSBlbW1lYW5zKG1tb2RlbDEsIH4gRXRobmljaXR5KQplZmZfc2l6ZShlbW1WMiwgc2lnbWEgPSB0b3RTRCwgZWRmID0gNSkKYGBgCgoKYGBge3J9CiNlZmZlY3Qgc2l6ZSBvZiBwcm9ncmFtbWUgcGFydAplbW1lYW5zKG1tb2RlbDEsIGxpc3QocGFpcndpc2V+UFJPR1JBTU1FX1BBUlQpLCBhZGp1c3Q9InR1a2V5IikKI2ludGVyZXN0aW5nIGFzIG9ubHkgc29tZSBwYXJ0cyBzaG93IHNpZyBkaWZmZXJlbmNlIG9uIG1hcmsuLi4gc29tZSBwYXJ0cyBlLmcuIEIsVCwgc2hvdyBubyBzaWcgZGlmZiBvbiBtYXJrPz8gd29uZGVyIHdoeT8/IGlzIHRoaXMgY29ycmVjdCBpbnRlcnByZXRhdGlvbj8/CmBgYAoKYGBge3J9CiNUaGVyZSB3YXMgYW4gaW50ZXJhY3Rpb24gc28gY29tcHV0aW5nIGVtbWVhbnMgZm9yIHRoaXMKZW1tZWFucyhtbW9kZWwxLCBwYWlyd2lzZSB+IEV0aG5pY2l0eSB8IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykKYGBgCgpgYGB7cn0KI2VmZmVjdCBzaXplIG9mIGludGVyYWN0aW9uIAplbW1WMyA8LSBlbW1lYW5zKG1tb2RlbDEsIH4gRXRobmljaXR5IHwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKQplZmZfc2l6ZShlbW1WMywgc2lnbWEgPSB0b3RTRCwgZWRmID0gNSkKYGBgCgoKIyMgRXZhbHVhdGluZyBtaXhlZCBtb2RlbC0xIGFzc3VtcHRpb25zCgojIyMgbGluZWFyaXR5ClVzdWFsbHksIHRoZXJlIGFyZSAzIG1vZGVsIGFzc3VtcHRpb25zIHdoaWNoIHdlIG5lZWQgdG8gY2hlY2s7IGxpbmVhcml0eSwgaG9tb3NjZWRhc3RpY2l0eSBhbmQgbm9ybWFsaXR5IG9mIHJlc2lkdWFscy4KV2UgZG8gbm90IG5lZWQgdG8gbG9vayBhdCBsaW5lYXJpdHkgYXMgYWxsIHZhcmlhYmxlcyBpbiB0aGlzIG1vZGVsIGFyZSBjYXRlZ29yaWNhbC4KCiMjIyBIb21vc2NlZGFzdGljaXR5OgpgYGB7cn0KI2NvZGUgYWRhcHRlZCBmcm9tIGh0dHBzOi8vYWRlbW9zLnBlb3BsZS51aWMuZWR1L0NoYXB0ZXIxOC5odG1sCnBsb3QobW1vZGVsMSkKYGBgCgpgYGB7cn0KI2JveHBsb3Qgb2YgcmVzaWR1YWxzIHRvIGFzc2VzcyBkaXN0cmlidXRpb24gb2YgcmVzaWR1YWxzCmJveHBsb3QobW1vZGVsMUByZXNwJHd0cmVzfmZ1bGxfZGF0YSRBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpCmBgYAoKYGBge3J9CmJveHBsb3QobW1vZGVsMUByZXNwJHd0cmVzfmZ1bGxfZGF0YSRFdGhuaWNpdHkpCmBgYAoKYGBge3J9CmJveHBsb3QobW1vZGVsMUByZXNwJHd0cmVzfmZ1bGxfZGF0YSRQUk9HUkFNTUVfUEFSVCkKYGBgCkFub3RoZXIgd2F5IHRvIGNoZWNrIGZvciBob21vc2NlZGFzdGljaXR5IC0gKHNkKSByZXNpZHVhbHMKYGBge3J9CiNBU1NFU1NNRU5UIFRZUEUKbW1vZGVsMXJlczwtbW1vZGVsMUByZXNwJHd0cmVzCiNhc3Nlc3NtZW50IHR5cGUgcmVzaWR1YWxzIGluIGRmCm1tb2RlbDFkZjwtZGF0YS5mcmFtZShmdWxsX2RhdGEkQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLG1tb2RlbDFyZXMpIApuYW1lcyhtbW9kZWwxZGYpPC1jKCJBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MiLCJyZXNpZHVhbCIpCm1tb2RlbDFkZiAlPiUgZ3JvdXBfYnkoQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSU+JSBzdW1tYXJpc2Uoc2QocmVzaWR1YWwpKSAKYGBgCgpTaW5jZSB0aGUgaGlnaGVzdCBzZCBpcyBsZXNzIHRoYW4gdHdpY2UgdGhlIGxvd2VzdCwgdGhhdCBpcyBmaW5lIChhY2NvcmRpbmcgdG8gZ2VuZXJhbCBydWxlIG9mIHRodW1iIGZvciBob21vc2NlZGFzdGljaXR5IGFjcm9zcyBncm91cHMgb2YgYSBmYWN0b3IpCgpOb3cgd2Ugd2lsbCBjaGVjayBmb3IgdGhlIHJlbWFpbmluZyB2YXJpYWJsZXMuLi4KYGBge3J9CiNFdGhuaWNpdHkKbW1vZGVsMWRmMjwtZGF0YS5mcmFtZShmdWxsX2RhdGEkRXRobmljaXR5LG1tb2RlbDFyZXMpIApuYW1lcyhtbW9kZWwxZGYyKTwtYygiRXRobmljaXR5IiwicmVzaWR1YWwiKQptbW9kZWwxZGYyICU+JSBncm91cF9ieShFdGhuaWNpdHkpJT4lIHN1bW1hcmlzZShzZChyZXNpZHVhbCkpIAoKI2Fzc3VtcHRpb24gc2F0aXNmaWVkIApgYGAKCmBgYHtyfQojcHJvZ3JhbW1lIHBhcnQKbW1vZGVsMWRmMzwtZGF0YS5mcmFtZShmdWxsX2RhdGEkUFJPR1JBTU1FX1BBUlQsbW1vZGVsMXJlcykgCm5hbWVzKG1tb2RlbDFkZjMpPC1jKCJQcm9ncmFtbWVfUGFydCIsInJlc2lkdWFsIikKbW1vZGVsMWRmMyAlPiUgZ3JvdXBfYnkoUHJvZ3JhbW1lX1BhcnQpJT4lIHN1bW1hcmlzZShzZChyZXNpZHVhbCkpCgojYXNzdW1wdGlvbiBtZXQgZm9yIHRoaXMgY2F0ZWdvcmljYWwgdmFyaWFibGUgYXMgdGhlIGxhcmdlc3QgcmVzaWR1YWwgaXMgMTUuMyBhbmQgaXMgbGVzcyB0aGFuIDIqIHRoZSBzbWFsbGVzdCByZXNpZHVhbCBvZiBEICg5LjIwKQpgYGAKIyMjIG5vcm1hbGl0eSBvZiByZXNpZHVhbHMKCmBgYHtyfQpxcW5vcm0obW1vZGVsMXJlcykKYGBgCgojIyMgQ29uZmlkZW5jZSBpbnRlcnZhbHMKYGBge3J9CiNJbnRlcnZhbCBlc3RpbWF0ZXMgZm9yIG1peGVkIG1vZGVscwojY29uZmludChtbW9kZWwxKQpgYGAKClRoZSBhYm92ZSBtb2RlbCBpcyBhc3Nlc3NpbmcgZWZmZWN0IG9mIHZhcmlhYmxlcyBvbiBzdHVkZW50IHBlcmZvcm1hbmNlIGJ5IGxvb2tpbmcgYXQgYXZlcmFnZSBtYXJrcy4gV2Ugd2FudCB0byBsb29rIGF0IGF3YXJkaW5nIGdhcHMgc28gd2lsbCBjYXRlZ29yaXNlIHRoZSBtYXJrcyBpbnRvIHRvcCBkZWdyZWUgYW5kIGJlbG93IGFuZCBydW4gYSBiaW5vbWlhbCBtb2RlbC4KCiMjIEdlbmVyYWxpc2VkIGxpbmVhciBtaXhlZCBtb2RlbCAxLjIKCmBgYHtyfQojY3JlYXRpbmcgYmlub21pYWwgcmVzcG9uc2UgdmFyaWFibGUKZnVsbF9kYXRhICU+JQogIGdyb3VwX2J5KElEMikgJT4lCiAgbXV0YXRlKEF2ZXJhZ2VfbWFyayA9IG1lYW4oTUFSSykpICU+JQogIG11dGF0ZShHcmFkZSA9IGNhc2Vfd2hlbihBdmVyYWdlX21hcmsgPj0gNjAgfiAxLAogICAgICAgICAgICAgICAgICAgICAgICAgICBBdmVyYWdlX21hcmsgPCA2MCB+IDApKSAtPiBmdWxsX2RhdGEzCgpgYGAKCmBgYHtyfQojMS4JV2hhdCBhZmZlY3QgZG9lcyB0aGUgYXNzZXNzbWVudCB0eXBlIGhhdmUgb24gdGhlIEJBTUUgYXdhcmRpbmcgZ2FwPwpsaW5tb2Q8LWdsbWVyKGFzLmZhY3RvcihHcmFkZSkgfiBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MgKiBFdGhuaWNpdHkgKyBQUk9HUkFNTUVfUEFSVCArICgxfElEMikgKyAoMXxNT0RVTEVfQ09ERSksIGZhbWlseT0iYmlub21pYWwiLGRhdGEgPSBmdWxsX2RhdGEzLCBuQUdRPTApCmBgYAoKYGBge3J9CnN1bW1hcnkobGlubW9kKQpgYGAKCmBgYHtyfQojY2FsY3VsYXRpbmcgb2RkcyByYXRpb3MKc2UgPC0gc3FydChkaWFnKHZjb3YobGlubW9kKSkpCiMgdGFibGUgb2YgZXN0aW1hdGVzIHdpdGggOTUlIENJCih0YWIgPC0gY2JpbmQoRXN0ID0gZml4ZWYobGlubW9kKSwgTEwgPSBmaXhlZihsaW5tb2QpIC0gMS45NiAqIHNlLCBVTCA9IGZpeGVmKGxpbm1vZCkgKyAxLjk2ICoKICAgIHNlKSkKYGBgCgpgYGB7cn0KI29kZCByYXRpb3MKZXhwKHRhYikKYGBgCgpgYGB7cn0KI2V2YWx1YXRpbmcgbW9kZWwgc2lnbmlmaWNhbmNlL3F1YWxpdHkgCiNBbmFseXNpcyBvZiBEZXZpYW5jZSB0ZXN0CkFub3ZhKGxpbm1vZCkKYGBgCgpgYGB7cn0KI2ludGVyYWN0aW9uIHBsb3QgZnJvbSBtb2RlbCAxLjIKcGxvdF9tb2RlbChsaW5tb2QsIHR5cGUgPSAicHJlZCIsIHRlcm1zID0gYygiRXRobmljaXR5IiwgIkFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyIpKQpgYGAKCmBgYHtyfQplbW1lYW5zKGxpbm1vZCwgfiBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIHR5cGUgPSAicmVzcG9uc2UiKQpgYGAKCmBgYHtyfQplbW1lYW5zKGxpbm1vZCwgfiBFdGhuaWNpdHksIHR5cGUgPSAicmVzcG9uc2UiKQpgYGAKCgpgYGB7cn0KZW1tZWFucyhsaW5tb2QsIH4gRXRobmljaXR5IHwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCB0eXBlID0gInJlc3BvbnNlIikKYGBgCgpUaGlzIG1vZGVsIGRvZXMgbm90IHJlcXVpcmUgY2hlY2tpbmcgb2YgYXNzdW1wdGlvbnMKCiMgKipSZXNlYXJjaCBRdWVzdGlvbiAyKioKIyAqKjIuCVdoYXQgYWZmZWN0IGRvZXMgdGhlIGFzc2Vzc21lbnQgdHlwZSBoYXZlIG9uIHRoZSBwZXJmb3JtYW5jZSBvZiBOb24tV2hpdGUgc3R1ZGVudHMgKE5hbWVseSBBc2lhbiwgQmxhY2ssIENoaW5lc2UsIGFuZCBtaXhlZCkgYW5kIHRoZSBhd2FyZGluZyBnYXA/KioKCkRhdGEgd3JhbmdsaW5nOgpUaGlzIHJlc2VhcmNoIHF1ZXN0aW9uIHJlcXVpcmVkIGRpZmZlcmVudCBjYXRlZ29yaXNpc2F0aW9uIG9mIGV0aG5pY2l0aWVzLiAKYGBge3J9CiNjYXRlZ29yaXNpbmcgZXRobmljaXRpZXMgaW50byBBc2lhbiwgQmxhY2ssIENoaW5lc2UsIE1peGVkICYgV2hpdGUKCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIldoaXRlIC0gQnJpdGlzaCJdIDwtICJXaGl0ZSIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiV2hpdGUgLSBJcmlzaCJdIDwtICJXaGl0ZSIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiV2hpdGUgLSBTY290dGlzaCJdIDwtICJXaGl0ZSIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiT3RoZXIgV2hpdGUgQmFja2dyb3VuZCJdIDwtICJXaGl0ZSIKCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIkFzaWFuIE9yIEFzaWFuIEJyaXRpc2ggLSBCYW5nbGFkZXNoaSJdIDwtICJBc2lhbiIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiQXNpYW4gT3IgQXNpYW4gQnJpdGlzaCAtIEluZGlhbiJdIDwtICJBc2lhbiIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiQXNpYW4gT3IgQXNpYW4gQnJpdGlzaCAtIFBha2lzdGFuaSJdIDwtICJBc2lhbiIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiQmxhY2sgT3IgQmxhY2sgQnJpdGlzaCAtIEFmcmljYW4iXSA8LSAiQmxhY2siCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIkJsYWNrIE9yIEJsYWNrIEJyaXRpc2ggLSBDYXJpYmJlYW4iXSA8LSAiQmxhY2siCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIkNoaW5lc2UiXSA8LSAiQ2hpbmVzZSIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiTWl4ZWQgLSBXaGl0ZSBBbmQgQXNpYW4iXSA8LSAiTWl4ZWQiCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIk1peGVkIC0gV2hpdGUgQW5kIEJsYWNrIEFmcmljYW4iXSA8LSAiTWl4ZWQiCmNsZWFuX2V0aG5pY2l0eV9kYXRhMltjbGVhbl9ldGhuaWNpdHlfZGF0YTIgPT0gIk1peGVkIC0gV2hpdGUgQW5kIEJsYWNrIENhcmliYmVhbiJdIDwtICJNaXhlZCIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiT3RoZXIgQXNpYW4gQmFja2dyb3VuZCJdIDwtICJBc2lhbiIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiT3RoZXIgQmxhY2sgQmFja2dyb3VuZCJdIDwtICJCbGFjayIKY2xlYW5fZXRobmljaXR5X2RhdGEyW2NsZWFuX2V0aG5pY2l0eV9kYXRhMiA9PSAiT3RoZXIgTWl4ZWQgQmFja2dyb3VuZCJdIDwtICJNaXhlZCIKCiNyZW1vdmUgbm90IGtub3duLG5vIGluZm9ybWF0aW9uICYgaW5mbyByZWZ1c2VkLSBhcyBjb3VsZCBiZSBhbnkgZXRobmljaXR5LgojYWxzbyByZW1vdmVkIGd5cHN5IGFzIG9ubHkgMSBzdHVkZW50LSBub3QgZ29vZCBmb3IgcmVwcmVzZW50YXRpb24KI2Fsc28gcmVtb3ZlZCBvdGhlciBldGhuaWMgYmFja2dyb3VuZC0gYXMgbm90IHN1cmUgd2hhdCBjYXRlZ29yeSB0aGlzIHJlbGF0ZXMgdG8gCgpjbGVhbl9ldGhuaWNpdHlfZGF0YTI8LWNsZWFuX2V0aG5pY2l0eV9kYXRhMlshKGNsZWFuX2V0aG5pY2l0eV9kYXRhMiRFdGhuaWNpdHk9PSJJbmZvcm1hdGlvbiBSZWZ1c2VkIiB8IGNsZWFuX2V0aG5pY2l0eV9kYXRhMiRFdGhuaWNpdHk9PSJJbmZvcm1hdGlvbiBSZWZ1c2VkIC0gT0JTT0xFVEUgVkFMVUUiIHwgY2xlYW5fZXRobmljaXR5X2RhdGEyJEV0aG5pY2l0eT09Ik5vIEluZm9ybWF0aW9uIiB8IGNsZWFuX2V0aG5pY2l0eV9kYXRhMiRFdGhuaWNpdHk9PSJOb3QgS25vd24iIHwgY2xlYW5fZXRobmljaXR5X2RhdGEyJEV0aG5pY2l0eT09Ikd5cHN5IG9yIFRyYXZlbGxlciIgfCBjbGVhbl9ldGhuaWNpdHlfZGF0YTIkRXRobmljaXR5PT0iT3RoZXIgRXRobmljIEJhY2tncm91bmQiIHwgY2xlYW5fZXRobmljaXR5X2RhdGEyJEV0aG5pY2l0eT09IkFyYWIiKSAsXQpgYGAKCmBgYHtyfQojY29udmVydGluZyB0byBmYWN0b3JzCmNsZWFuX2V0aG5pY2l0eV9kYXRhMiAlPiUKICBtdXRhdGVfYXQodmFycyhHZW5kZXIsIEV0aG5pY2l0eSksIGxpc3QoZmFjdG9yKSkgLT4gY2xlYW5fZXRobmljaXR5X2RhdGEyCgpzdW1tYXJ5KGNsZWFuX2V0aG5pY2l0eV9kYXRhMikKYGBgCgpgYGB7cn0KI0V0aG5pY2l0eQojUGVyY2VudGFnZSBvZiBzdHVkZW50cyBiZWxvbmdpbmcgdG8gZWFjaCBldGhuaWNpdHkgYWZ0ZXIgY2F0ZWdvcmlzYXRpb24KY2xlYW5fZXRobmljaXR5X2RhdGEyICU+JQogIGdyb3VwX2J5KEV0aG5pY2l0eSkgJT4lCiAgc3VtbWFyaXNlKGNvdW50cyA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSAoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgZ2dwbG90KCAuLCBhZXMoeD0iIiwgeT1mcmVxLCBmaWxsPUV0aG5pY2l0eSkpICsKICBnZW9tX2JhcihzdGF0PSJpZGVudGl0eSIsIHdpZHRoPTEsIGNvbG9yPSJ3aGl0ZSIpICsKICBjb29yZF9wb2xhcigieSIsIHN0YXJ0PTApICsKICB0aGVtZV92b2lkKCkgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgY29sb3IgPSAid2hpdGUiLCBwb3NpdGlvbiA9IHBvc2l0aW9uX3N0YWNrKHZqdXN0ID0gMC41KSwgc2l6ZSA9IDIuNSkgKwogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiUGFpcmVkIikgKwogIGdndGl0bGUoIlBlcmNlbnRhZ2Ugb2Ygc3R1ZGVudHMgaW4gZWFjaCBldGhuaWMgY2F0ZWdvcnkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpIApgYGAKCkpvaW5pbmcgdHdvIGRhdGFzZXRzCmBgYHtyfQojSk9JTiBFVEhOSUMgREFUQSBBTkQgTUFSSyBEQVRBCmlubmVyX2pvaW4oY2xlYW5fZXRobmljaXR5X2RhdGEyLCBjbGVhbl9tYXJrX2RhdGEpIC0+IGZ1bGxfZGF0YTIKcHJpbnQoZGlzdGluY3QoZnVsbF9kYXRhMikpCmBgYAoKCiMjIDIuMSBFZmZlY3Qgb2YgRXRobmNpY2l0eSAoTm9uLVdoaXRlIHN0dWRlbnRzKSBvbiBzdHVkZW50IHBlcmZvcm1hbmNlCgpgYGB7cn0KYm94cGxvdChNQVJLfkV0aG5pY2l0eSwgZGF0YSA9IGZ1bGxfZGF0YTIpCmBgYAoKYGBge3J9CiNWaXN1YWxpc2UgZWZmZWN0IG9mIGV0aG5pY2l0eSBvbiBvdmVyYWxsL21lYW4gbWFyayBiZXR3ZWVuIDIwMTctMjAyMQpmdWxsX2RhdGEyICU+JQogIHNlbGVjdChFdGhuaWNpdHksIE1BUkspICU+JQogIGdyb3VwX2J5KEV0aG5pY2l0eSkgJT4lCiAgdGFsbHkobWVhbihNQVJLKSkgICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICBnZ3Bsb3QoYWVzKHg9RXRobmljaXR5LCB5ID0gQXZlcmFnZV9tYXJrLCBmaWxsID1FdGhuaWNpdHkpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIHBvc2l0aW9uID0gJ2RvZGdlJykgKyAKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlBhaXJlZCIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcm91bmQoQXZlcmFnZV9tYXJrLCAyKSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiQXZlcmFnZSBUZXN0IFNjb3JlcyBieSBFdGhuaWNpdHkgYmV0d2VlbiAyMDE3LTIwMjEiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB5bGFiKCdBdmVyYWdlIE1hcmsnKSAKYGBgCgpgYGB7cn0KI2xvb2tpbmcgYXQgYXZnIGdyYWRlIGZvciBlYWNoIGV0aG5pY2l0eSB0eXBlIGJldHdlZW4gMjAxNy0yMDIxCmZ1bGxfZGF0YTIgJT4lCiAgc2VsZWN0KElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIEV0aG5pY2l0eSxNQVJLKSAlPiUKICBncm91cF9ieShJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBFdGhuaWNpdHkpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICAjc3ByZWFkKGtleSA9IEV0aG5pY2l0eSwgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIGdncGxvdChhZXMoeCA9IElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIHkgPSBBdmVyYWdlX21hcmssIGNvbCA9IEV0aG5pY2l0eSkpICsKICBnZW9tX2xpbmUoKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZ3RpdGxlKCJBdmVyYWdlIGdyYWRlIGZvciBXaGl0ZSB2cyBCQU1FICgyMDE3LTIwMjEpIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeGxhYignQWNhZGVtaWMgWWVhcicpICsKICB5bGFiKCdBdmVyYWdlIE1hcmsnKSArCiAgbGFicyhjb2w9J0V0aG5pY2l0eScpIApgYGAKCiMjIyBBbmFseXNpcyBvZiBOb24td2hpdGUgc3R1ZGVudHMgQXdhcmRpbmcgR2FwICgyMDE3LTIwMjEpCgpgYGB7cn0KI1Zpc3VhbGlzaW5nIHBlcmNlbnRhZ2Ugb2Ygd2hpdGUgdnMgQkFNRSBzdHVkZW50cyBhY2hpZXZpbmcgdG9wIGRlZ3JlZSAoMjAxNy0yMDIxKQpmdWxsX2RhdGEyICU+JQogIGdyb3VwX2J5KElEMikgJT4lCiAgbXV0YXRlKEF2ZXJhZ2VfbWFyayA9IG1lYW4oTUFSSykpICU+JQogIGZpbHRlcihBdmVyYWdlX21hcmsgPj02MCkgJT4lCiAgc2VsZWN0KElEMiwgRXRobmljaXR5LCBBdmVyYWdlX21hcmspICU+JQogIGRpc3RpbmN0KCkgJT4lCiAgZ3JvdXBfYnkoRXRobmljaXR5KSAlPiUKICBzdW1tYXJpc2UoY291bnRzID0gbigpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgZ2dwbG90KGFlcyh4PUV0aG5pY2l0eSwgeSA9IGNvdW50cywgZmlsbCA9RXRobmljaXR5KSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYXN0ZWwxIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb24gPSBwb3NpdGlvbl9zdGFjayh2anVzdCA9IDEuMSksIHNpemUgPSAzKSAgKwogIGdndGl0bGUoIlBlcmNlbnRhZ2Ugb2Ygd2hpdGUgdnMgQkFNRSBzdHVkZW50cyBhY2hpZXZpbmcgdG9wIGRlZ3JlZSAoMjAxNy0yMDIxKSIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHlsYWIoJ051bWJlciBvZiBzdHVkZW50cycpCmBgYAoKYGBge3J9CiNCQU1FIEFXQVJESU5HIEdBUApmdWxsX2RhdGEyICU+JQogIGdyb3VwX2J5KElEMikgJT4lCiAgbXV0YXRlKEF2ZXJhZ2VfbWFyayA9IG1lYW4oTUFSSykpICU+JQogIG11dGF0ZShHcmFkZSA9IGNhc2Vfd2hlbihBdmVyYWdlX21hcmsgPj0gNzAgfiAiRmlyc3QgQ2xhc3MiLAogICAgICAgICAgICAgICAgICAgICAgICAgICBBdmVyYWdlX21hcmsgPj0gNjAgfiAiVXBwZXIgU2Vjb25kIENsYXNzIEhvbm91ciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIEF2ZXJhZ2VfbWFyayA+PSA1MCB+ICJMb3dlciBTZWNvbmQgQ2xhc3MgSG9ub3VyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgQXZlcmFnZV9tYXJrID49IDQwIH4gIlRoaXJkIGNsYXNzL3Bhc3MiLAogICAgICAgICAgICAgICAgICAgICAgICAgICBBdmVyYWdlX21hcmsgPCA0MCB+ICJGYWlsIikpICU+JQogIGdyb3VwX2J5KEV0aG5pY2l0eSwgR3JhZGUpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShmcmVxID0gKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgbXV0YXRlKHBlcmNlbnRhZ2UgPSBzY2FsZXM6OmxhYmVsX3BlcmNlbnQoKShjb3VudHMgLyBzdW0oY291bnRzKSkpICU+JQogIGdncGxvdChhZXMoeD1FdGhuaWNpdHksIHkgPSBmcmVxLCBmaWxsID1HcmFkZSkpICsKICBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JykgKyAKICBjb29yZF9mbGlwKCkgKwogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiUGFzdGVsMSIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcGVyY2VudGFnZSksIHBvc2l0aW9uID0gcG9zaXRpb25fc3RhY2sodmp1c3QgPSAwLjUpLCBzaXplID0gMi41KSArCiAgZ2d0aXRsZSgiRGlzdHJpYnV0aW9uIG9mIGRlZ3JlZSBvdXRjb21lcyBieSBldGhuaWMgZ3JvdXAgKDIwMTctMjAyMSkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICBsYWJzKHkgPSAiRnJlcXVlbmN5IiwgZmlsbCA9ICJEZWdyZWUgQ2xhc3NpZmljYXRpb24iKQpgYGAKCiMjIDIuMiBBbmFseXNpbmcgZWZmZWN0IG9mIGFzc2Vzc21lbnQgdHlwZSBvbiBub24gd2hpdGUgc3R1ZGVudHMgcGVyZm9ybWFuY2UgYW5kIGF3YXJkaW5nIGdhcAoKYGBge3J9CmZ1bGxfZGF0YTIgJT4lCiAgc2VsZWN0KEV0aG5pY2l0eSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLE1BUkspICU+JQogIGdyb3VwX2J5KEV0aG5pY2l0eSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICB0YWxseShtZWFuKE1BUkspKSAgJT4lCiAgcmVuYW1lKEF2ZXJhZ2VfbWFyayA9IG4pICU+JQogICAjc3ByZWFkKGtleSA9IEV0aG5pY2l0eSwgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIGdncGxvdChhZXMoeD1BU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIHkgPSBBdmVyYWdlX21hcmssIGZpbGwgPSBFdGhuaWNpdHkpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIHBvc2l0aW9uID0gJ2RvZGdlJykgICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJTZXQxIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSByb3VuZChBdmVyYWdlX21hcmssIDIpKSwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9MC45KSwgdmp1c3Q9LTAuMjUpICsKICBnZ3RpdGxlKCJBdmVyYWdlIE1hcmsgQnkgQXNzZXNzbWVudCBUeXBlIEFuZCBFdGhuaWNpdHkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBc3Nlc3NtZW50IFR5cGUnKSArCiAgeWxhYignQXZlcmFnZSBNYXJrJykKYGBgCgpFZmZlY3Qgb2YgYXNzZXNzbWVudCB0eXBlIG9uIGF3YXJkaW5nIGdhcCBmb3IgTm9uLXdoaXRlIGNhdGVnb3Jpc2UuIApgYGB7cn0KZnVsbF9kYXRhMiAlPiUKICBncm91cF9ieShJRDIpICU+JQogIG11dGF0ZShBdmVyYWdlX21hcmsgPSBtZWFuKE1BUkspKSAlPiUKICBmaWx0ZXIoQXZlcmFnZV9tYXJrID49NjApICU+JQogIHNlbGVjdChJRDIsIEV0aG5pY2l0eSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCBBdmVyYWdlX21hcmspICU+JQogIGRpc3RpbmN0KCkgJT4lCiAgc2VsZWN0KEV0aG5pY2l0eSwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBncm91cF9ieShFdGhuaWNpdHksQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKSAlPiUKICBzdW1tYXJpc2UoY291bnRzID0gbigpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgI3NlbGVjdChFdGhuaWNpdHksIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgcGVyY2VudGFnZSkgJT4lCiAgZ2dwbG90KGFlcyh4PUFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQywgeSA9IHBlcmNlbnRhZ2UsIGZpbGwgPSBFdGhuaWNpdHkpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIHBvc2l0aW9uID0gJ2RvZGdlJykgICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJTZXQxIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGg9MC45KSwgdmp1c3Q9LTAuMjUpICsKICBnZ3RpdGxlKCJFZmZlY3Qgb2YgYXNzZXNzbWVudCB0eXBlIG9uIGF3YXJkaW5nIGdhcCIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ0Fzc2Vzc21lbnQgVHlwZScpICsKICB5bGFiKCdQZXJjZW50YWdlIG9mIHN0dWRlbnRzJykKYGBgCgpgYGB7cn0KaW50ZXJhY3Rpb24ucGxvdCh4LmZhY3RvciA9IGZ1bGxfZGF0YTIkRXRobmljaXR5LCAjeC1heGlzIHZhcmlhYmxlCiAgICAgICAgICAgICAgICAgdHJhY2UuZmFjdG9yID0gZnVsbF9kYXRhMiRBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsICN2YXJpYWJsZSBmb3IgbGluZXMKICAgICAgICAgICAgICAgICByZXNwb25zZSA9IGZ1bGxfZGF0YTIkTUFSSywgI3ktYXhpcyB2YXJpYWJsZQogICAgICAgICAgICAgICAgIGZ1biA9IG1lZGlhbiwgI21ldHJpYyB0byBwbG90CiAgICAgICAgICAgICAgICAgeWxhYiA9ICJNYXJrIiwKICAgICAgICAgICAgICAgICB4bGFiID0gIkV0aG5pY2l0eSIsCiAgICAgICAgICAgICAgICAgY29sID0gYygicGluayIsICJsaWdodGJsdWUiKSwKICAgICAgICAgICAgICAgICBsdHkgPSAxLCAjbGluZSB0eXBlCiAgICAgICAgICAgICAgICAgbHdkID0gMiwgI2xpbmUgd2lkdGgKICAgICAgICAgICAgICAgICB0cmFjZS5sYWJlbCA9ICJBc3Nlc3NtZW50IHR5cGUiKQpgYGAKCiMgU3RhdGlzdGljYWwgQW5hbHlzaXMKCiMjIE1peGVkIE1vZGVsIDIKYGBge3J9CiNRdWVzdGlvbiAyOgojCTIuCVdoYXQgYWZmZWN0IGRvZXMgdGhlIGFzc2Vzc21lbnQgdHlwZSBoYXZlIG9uIHRoZSBwZXJmb3JtYW5jZSBvZiBOb24tV2hpdGUgc3R1ZGVudHMgKE5hbWVseSBBc2lhbiwgQmxhY2ssIENoaW5lc2UsIGFuZCBtaXhlZCkKCm1tb2RlbDI8LSBsbWVyKE1BUksgfiBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MgKiBFdGhuaWNpdHkgKyBQUk9HUkFNTUVfUEFSVCArICgxfElEMikgKyAoMXxNT0RVTEVfQ09ERSksIGRhdGEgPSBmdWxsX2RhdGEyKQoKYGBgCgpgYGB7cn0Kc3VtbWFyeShtbW9kZWwyKQojRnVsbCBtb2RlbCByZXN1bHRzCmBgYAoKYGBge3J9CiNtb2RlbCAyIGludGVyYWN0aW9uIHBsb3QKcGxvdF9tb2RlbChtbW9kZWwyLCB0eXBlID0gInByZWQiLCB0ZXJtcyA9IGMoIkV0aG5pY2l0eSIsICJBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MiKSkKYGBgCgoKYGBge3J9CiNBbm92YSB0ZXN0CmFub3ZhKG1tb2RlbDIpCmBgYAoKU2ltaWxhciB0byBSUTEsIGFsbCB2YXJpYWJsZXMgaG9sZCBhIHNpZ25pZmljYW50IGVmZmVjdCBvbiBzdHVkZW50IHBlcmZvcm1hbmNlCgpgYGB7cn0KI2VmZmVjdCBzaXplCmVmZmVjdHNpemU6OmV0YV9zcXVhcmVkKG1tb2RlbDIsIHBhcnRpYWwgPSBUUlVFKQpgYGAKCmBgYHtyfQojZW1tZWFucyBpcyBtZWFudCB0byBiZSBsb29rZWQgYXQgZm9yIG1haW4gZWZmZWN0LCBpbiB0aGlzIGNhc2UgdGhleSB3ZXJlIGFsbCBzaWduaWZpY2FudCBzbyB3aWxsIGRvIGZvciBlYWNoCmVtbWVhbnMobW1vZGVsMiwgbGlzdChwYWlyd2lzZX5BU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpLCBhZGp1c3Q9InR1a2V5IikKYGBgCgpgYGB7cn0KZW1tZWFucyhtbW9kZWwyLCBsaXN0KHBhaXJ3aXNlfkV0aG5pY2l0eSksIGFkanVzdD0idHVrZXkiKQpgYGAKCmBgYHtyfQplbW1lYW5zKG1tb2RlbDIsIGxpc3QocGFpcndpc2V+UFJPR1JBTU1FX1BBUlQpLCBhZGp1c3Q9InR1a2V5IikKYGBgCgpgYGB7cn0KI1RoZXJlIHdhcyBhbiBpbnRlcmFjdGlvbiBzbyBuZWVkIGVtbWVhbnMgZm9yIHRoaXMKZW1tZWFucyhtbW9kZWwyLCBwYWlyd2lzZSB+IEV0aG5pY2l0eSB8IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykKYGBgCgojIyBFdmFsdWF0aW5nIG1peGVkIG1vZGVsIDIgYXNzdW1wdGlvbnMKCiMjIyBIb21vZHNjZWRhc3RpY2l0eSAKCmBgYHtyfQojcGxvdCBvZiBtaXhlZCBtb2RlbCAyCnBsb3QobW1vZGVsMikKYGBgCgpgYGB7cn0KYm94cGxvdChtbW9kZWwyQHJlc3Akd3RyZXN+ZnVsbF9kYXRhMiRBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpCmBgYAoKYGBge3J9CmJveHBsb3QobW1vZGVsMkByZXNwJHd0cmVzfmZ1bGxfZGF0YTIkRXRobmljaXR5KQpgYGAKCmBgYHtyfQpib3hwbG90KG1tb2RlbDJAcmVzcCR3dHJlc35mdWxsX2RhdGEyJFBST0dSQU1NRV9QQVJUKQpgYGAKQW5vdGhlciB3YXkgdG8gY2hlY2sgZm9yIGhvbWVvc2NlZGFzdGljaXR5IC0gKHNkKSByZXNpZHVhbHMKYGBge3J9CiNBU1NFU1NNRU5UIFRZUEUKbW1vZGVsMnJlczwtbW1vZGVsMkByZXNwJHd0cmVzCiNhc3Nlc3NtZW50IHR5cGUgcmVzaWR1YWxzIGluIGRmCm1tb2RlbDJkZjwtZGF0YS5mcmFtZShmdWxsX2RhdGEyJEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyxtbW9kZWwycmVzKSAKbmFtZXMobW1vZGVsMmRmKTwtYygiQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDIiwicmVzaWR1YWwiKQptbW9kZWwyZGYgJT4lIGdyb3VwX2J5KEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyklPiUgc3VtbWFyaXNlKHNkKHJlc2lkdWFsKSkgCmBgYApUaGUgYXNzdW1wdGlvbiBpcyBtZXQgYmVjYXVzZSBoaWdoZXN0IHNkIGlzIGxlc3MgdGhhbiB0d2ljZSB0aGUgbG93ZXN0LiAKYGBge3J9CiNFdGhuaWNpdHkKbW1vZGVsMmRmMjwtZGF0YS5mcmFtZShmdWxsX2RhdGEyJEV0aG5pY2l0eSxtbW9kZWwycmVzKSAKbmFtZXMobW1vZGVsMmRmMik8LWMoIkV0aG5pY2l0eSIsInJlc2lkdWFsIikKbW1vZGVsMmRmMiAlPiUgZ3JvdXBfYnkoRXRobmljaXR5KSU+JSBzdW1tYXJpc2Uoc2QocmVzaWR1YWwpKSAKYGBgCkFnYWluLCB0aGUgYXNzdW1wdGlvbiBpcyBtZXQuIAoKYGBge3J9CiNQcm9ncmFtbWUgcGFydAptbW9kZWwyZGYzPC1kYXRhLmZyYW1lKGZ1bGxfZGF0YTIkUFJPR1JBTU1FX1BBUlQsbW1vZGVsMnJlcykgCm5hbWVzKG1tb2RlbDJkZjMpPC1jKCJQcm9ncmFtbWVfUGFydCIsInJlc2lkdWFsIikKbW1vZGVsMmRmMyAlPiUgZ3JvdXBfYnkoUHJvZ3JhbW1lX1BhcnQpJT4lIHN1bW1hcmlzZShzZChyZXNpZHVhbCkpCmBgYApBc3N1bXB0aW9uIHNhdGlzZmllZCBhZ2Fpbi4uCgpNb3Zpbmcgb250byBjaGVja2luZyB0aGUgcmVzaWR1YWxzIG9mIHRoZSBtb2RlbCBhcmUgbm9ybWFsbHkgZGlzdHJpYnV0ZWQuCgojIyMgTm9ybWFsaXR5IG9mIHJlc2lkdWFscwoKYGBge3J9CiNub3cgY2hlY2sgZm9yIHJlc2lkdWFscyBmb3IgbW1vZGVsMgpxcW5vcm0obW1vZGVsMnJlcykKYGBgCiMjIyBDb25maWRlbmNlIEludGVydmFscwoKYGBge3J9CiNjb25maW50KG1tb2RlbDIpCmBgYAoKIyAqKlJlc2VhcmNoIFF1ZXN0aW9uIDMqKgojICoqMy4gSXMgdGhlcmUgYSBsaW5rIGJldHdlZW4gYXNzZXNzbWVudCB0eXBlLCBnZW5kZXIgYW5kIHN0dWRlbnQgYWNoaWV2ZW1lbnQ/KioKCkRhdGEgc2V0IGZvciB0aGlzIHF1ZXN0aW9uIHdpbGwgYmUgZnVsbF9kYXRhIGRhdGEgc2V0IGFzIGFscmVhZHkgaW5jbHVkZXMgR2VuZGVyIGRhdGEuIENyZWF0ZWQgdGhpcyBpbiBwcmVsaW1pbmFyeSBhbmFseXNpcy4KCiMjIDMuMSBFZmZlY3Qgb2YgR2VuZGVyIG9uIHN0dWRlbnQgcGVyZm9ybWFuY2UKCkRpc3RyaWJ1dGlvbiBvZiBNYXJrcyBvZiBNYWxlICYgRmVtYWxlIHN0dWRlbnRzICgyMDE3LTIwMjEpCgpgYGB7cn0KYm94cGxvdChNQVJLfkdlbmRlciwgZGF0YSA9IGZ1bGxfZGF0YSkKYGBgCkJlbG93IGlzIGJhciBjaGFydCBzaG93aW5nIG9mIEdlbmRlciBvZiBldGhuaWNpdHkgb24gbWVhbiBtYXJrIGJldHdlZW4gMjAxNy0yMDIxCgpgYGB7cn0KI1Zpc3VhbGlzZSBlZmZlY3Qgb2YgZXRobmljaXR5IG9uIG92ZXJhbGwvbWVhbiBtYXJrIGJldHdlZW4gMjAxNy0yMDIxCmZ1bGxfZGF0YSAlPiUKICBzZWxlY3QoR2VuZGVyLCBNQVJLKSAlPiUKICBncm91cF9ieShHZW5kZXIpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICAlPiUKICByZW5hbWUoQXZlcmFnZV9tYXJrID0gbikgJT4lCiAgZ2dwbG90KGFlcyh4PUdlbmRlciwgeSA9IEF2ZXJhZ2VfbWFyaywgZmlsbCA9R2VuZGVyKSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICsgCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYWlyZWQiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHJvdW5kKEF2ZXJhZ2VfbWFyaywgMikpLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjkpLCB2anVzdD0tMC4yNSkgKwogIGdndGl0bGUoIkF2ZXJhZ2UgVGVzdCBTY29yZXMgYnkgR2VuZGVyIGJldHdlZW4gMjAxNy0yMDIxIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeWxhYignQXZlcmFnZSBNYXJrJykgCmBgYAoKYGBge3J9CiNsb29raW5nIGF0IGF2ZyBncmFkZSBmb3IgZWFjaCBldGhuaWNpdHkgdHlwZSBiZXR3ZWVuIDIwMTctMjAyMQpmdWxsX2RhdGEgJT4lCiAgc2VsZWN0KElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIEdlbmRlcixNQVJLKSAlPiUKICBncm91cF9ieShJTlNUQU5DRV9BQ0FERU1JQ19ZRUFSLCBHZW5kZXIpICU+JQogIHRhbGx5KG1lYW4oTUFSSykpICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICAjc3ByZWFkKGtleSA9IEV0aG5pY2l0eSwgdmFsdWUgPSBBdmVyYWdlX21hcmspICU+JQogIGdncGxvdChhZXMoeCA9IElOU1RBTkNFX0FDQURFTUlDX1lFQVIsIHkgPSBBdmVyYWdlX21hcmssIGNvbCA9IEdlbmRlcikpICsKICBnZW9tX2xpbmUoKSArCiAgZ2VvbV9wb2ludCgpICsKICBnZ3RpdGxlKCJBdmVyYWdlIGdyYWRlIGZvciBNZW4gdnMgV29tZW4gKDIwMTctMjAyMSkiKSArCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdCA9IDAuNSkpICsKICB4bGFiKCdBY2FkZW1pYyBZZWFyJykgKwogIHlsYWIoJ0F2ZXJhZ2UgTWFyaycpICsKICBsYWJzKGNvbD0nR2VuZGVyJykgCmBgYAoKIyMjIEFuYWx5c2lzIG9mIEdlbmRlciBBd2FyZGluZyBHYXAgKDIwMTctMjAyMSkKCmBgYHtyfQojVmlzdWFsaXNpbmcgcGVyY2VudGFnZSBvZiB3aGl0ZSB2cyBCQU1FIHN0dWRlbnRzIGFjaGlldmluZyB0b3AgZGVncmVlICgyMDE3LTIwMjEpCmZ1bGxfZGF0YSAlPiUKICBncm91cF9ieShJRDIpICU+JQogIG11dGF0ZShBdmVyYWdlX21hcmsgPSBtZWFuKE1BUkspKSAlPiUKICBmaWx0ZXIoTUFSSyA+PTYwKSAlPiUKICBzZWxlY3QoSUQyLCBHZW5kZXIsIEF2ZXJhZ2VfbWFyaykgJT4lCiAgZGlzdGluY3QoKSAlPiUKICBncm91cF9ieShHZW5kZXIpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9R2VuZGVyLCB5ID0gY291bnRzLCBmaWxsID1HZW5kZXIpKSArCiAgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIHBvc2l0aW9uID0gJ2RvZGdlJykgKyAKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlBhc3RlbDEiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHBlcmNlbnRhZ2UpLCBwb3NpdGlvbiA9IHBvc2l0aW9uX3N0YWNrKHZqdXN0ID0gMS4xKSwgc2l6ZSA9IDMpICArCiAgZ2d0aXRsZSgiTnVtYmVyIG9mIE1lbiB2cyBXb21lbiBhY2hpZXZpbmcgdG9wIGRlZ3JlZSAoMjAxNy0yMDIxKSIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHlsYWIoJ051bWJlciBvZiBzdHVkZW50cycpCmBgYApUaGlzIGlzIHZlcnkgaW50ZXJlc3RpbmcgYXMgbW9yZSBtZW4gYXJlIGJlaW5nIGF3YXJkZWQgdG9wIGRlZ3JlZXMgYnV0IG9uIGF2ZXJhZ2Ugd29tYW4gcGVyZm9ybSBiZXR0ZXIuClRoaXMgbWF5IGJlIGJlY2F1c2UgbW9yZSBtZW4gaW4gdGhpcyBkYXRhc2V0IHRoZW4gd29tYW4uClNvLCBiZWxvdyBpcyBhIGJldHRlciBjb21wYXJpc29uLCBsb29raW5nIGF0IGRpc3RyaWJ1dGlvbiBvZiBncmFkZXMuCmBgYHtyfQojR2VuZGVyIEFXQVJESU5HIEdBUApmdWxsX2RhdGEgJT4lCiAgZ3JvdXBfYnkoSUQyKSAlPiUKICBtdXRhdGUoQXZlcmFnZV9tYXJrID0gbWVhbihNQVJLKSkgJT4lCiAgbXV0YXRlKEdyYWRlID0gY2FzZV93aGVuKEF2ZXJhZ2VfbWFyayA+PSA3MCB+ICJGaXJzdCBDbGFzcyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIEF2ZXJhZ2VfbWFyayA+PSA2MCB+ICJVcHBlciBTZWNvbmQgQ2xhc3MgSG9ub3VyIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgQXZlcmFnZV9tYXJrID49IDUwIH4gIkxvd2VyIFNlY29uZCBDbGFzcyBIb25vdXIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICBBdmVyYWdlX21hcmsgPj0gNDAgfiAiVGhpcmQgY2xhc3MvcGFzcyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIEF2ZXJhZ2VfbWFyayA8IDQwIH4gIkZhaWwiKSkgJT4lCiAgZ3JvdXBfYnkoR2VuZGVyLCBHcmFkZSkgJT4lCiAgc3VtbWFyaXNlKGNvdW50cyA9IG4oKSkgJT4lCiAgbXV0YXRlKGZyZXEgPSAoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICBtdXRhdGUocGVyY2VudGFnZSA9IHNjYWxlczo6bGFiZWxfcGVyY2VudCgpKGNvdW50cyAvIHN1bShjb3VudHMpKSkgJT4lCiAgZ2dwbG90KGFlcyh4PUdlbmRlciwgeSA9IGZyZXEsIGZpbGwgPUdyYWRlKSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknKSArIAogIGNvb3JkX2ZsaXAoKSArCiAgc2NhbGVfZmlsbF9icmV3ZXIocGFsZXR0ZSA9ICJQYXN0ZWwxIikgKwogIGdlb21fdGV4dChhZXMobGFiZWwgPSBwZXJjZW50YWdlKSwgcG9zaXRpb24gPSBwb3NpdGlvbl9zdGFjayh2anVzdCA9IDAuNSksIHNpemUgPSAyLjUpICsKICBnZ3RpdGxlKCJEaXN0cmlidXRpb24gb2YgZGVncmVlIG91dGNvbWVzIGJ5IGdlbmRlciAoMjAxNy0yMDIxKSIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIGxhYnMoeSA9ICJGcmVxdWVuY3kiLCBmaWxsID0gIkRlZ3JlZSBDbGFzc2lmaWNhdGlvbiIpCmBgYApUaGlzIGdyYXBoIGlzIG11Y2ggYmV0dGVyIGF0IHNob3dpbmcgd2hhdCBpcyBhY3R1YWxseSBnb2luZyBvbiBpbiByZWdhcmRzIHRvIHdoYXQgZGVncmVlIGNsYXNzaWZpY2F0aW9uIHN0dWRlbnRzIGFyZSBiZWluZyBhd2FyZGVkLgpUaGlzIG1ha2VzIG1vcmUgc2Vuc2Ugbm93Li4uClRoaXMgaXMgc2hvd2luZyAzMy41JSBvZiB3b21hbiBzdHVkZW50cyBhcmUgZ2V0dGluZyBhIGZpcnN0IGluIGNvbXBhcmlzb24gdG8gbWFsZSBzdHVkZW50cwpTbyBmZW1hbGVzIGRvIHBlcmZvcm0gYmV0dGVyLyBtb3JlIGZlbWFsZXMgYXJlIGF3YXJkZWQgYSB0b3AgZGVncmVlIGluIGNvbXBhcmlzb24gdG8gbWFsZSBzdHVkZW50cywgYnV0IHdoZW4gbG9va2luZyBhdCBvdmVyYWxsIGRlZ3JlZSBvdXRjb21lcywgbW9yZSBtYWxlcyBhcmUgZ2V0dGluZyB0b3AgZGVncmVlIGJlY2F1c2UgdGhlcmUgYXJlIG1vcmUgbWFsZSBzdHVkZW50cyBhdCB1bml2ZXJzaXR5LgoKIyMgMy4yIEFuYWx5c2luZyBlZmZlY3Qgb2YgYXNzZXNzbWVudCB0eXBlIGFuZCBnZW5kZXIgb24gc3R1ZGVudCBwZXJmb3JtYW5jZSBhbmQgYXdhcmRpbmcgZ2FwCgpgYGB7cn0KZnVsbF9kYXRhICU+JQogIHNlbGVjdChHZW5kZXIsIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyxNQVJLKSAlPiUKICBncm91cF9ieShHZW5kZXIsIEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykgJT4lCiAgdGFsbHkobWVhbihNQVJLKSkgICU+JQogIHJlbmFtZShBdmVyYWdlX21hcmsgPSBuKSAlPiUKICBnZ3Bsb3QoYWVzKHg9QVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCB5ID0gQXZlcmFnZV9tYXJrLCBmaWxsID0gR2VuZGVyKSkgKwogIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBwb3NpdGlvbiA9ICdkb2RnZScpICArIAogIHNjYWxlX2ZpbGxfYnJld2VyKHBhbGV0dGUgPSAiU2V0MSIpICsKICBnZW9tX3RleHQoYWVzKGxhYmVsID0gcm91bmQoQXZlcmFnZV9tYXJrLCAyKSksIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoPTAuOSksIHZqdXN0PS0wLjI1KSArCiAgZ2d0aXRsZSgiQXZlcmFnZSBNYXJrIEJ5IEFzc2Vzc21lbnQgVHlwZSBBbmQgR2VuZGVyIikgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3QgPSAwLjUpKSArCiAgeGxhYignQXNzZXNzbWVudCBUeXBlJykgKwogIHlsYWIoJ0F2ZXJhZ2UgTWFyaycpCmBgYApUaGlzIHNob3dzIHdvbWVuIHBlcmZvcm0gYmV0dGVyIHRoYW4gbWVuIGluIGJvdGggYXNzZXNzbWVudCB0eXBlcywgYnV0IHRoZSBkaWZmZXJlbmNlIGJ5IGFzc2Vzc21lbnQgdHlwZSBzZWVtcyBzaW1pbGFyLi4uIFNvIG9uZSBnZW5kZXIgZG9lcyBub3QgcGVyZm9ybSBzaWduaWZpY2FudGx5IGRpZmZlcmVudCB0aGFuIG90aGVyIGluIG9uZSBhc3Nlc3NtZW50IHR5cGUuIAoKYGBge3J9CiAgZnVsbF9kYXRhICU+JQogIGdyb3VwX2J5KElEMikgJT4lCiAgbXV0YXRlKEF2ZXJhZ2VfbWFyayA9IG1lYW4oTUFSSykpICU+JQogIGZpbHRlcihBdmVyYWdlX21hcmsgPj02MCkgJT4lCiAgc2VsZWN0KElEMiwgR2VuZGVyLCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MsIEF2ZXJhZ2VfbWFyaykgJT4lCiAgZGlzdGluY3QoKSAlPiUKICBzZWxlY3QoR2VuZGVyLCBBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIGdyb3VwX2J5KEdlbmRlcixBU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpICU+JQogIHN1bW1hcmlzZShjb3VudHMgPSBuKCkpICU+JQogIG11dGF0ZShwZXJjZW50YWdlID0gc2NhbGVzOjpsYWJlbF9wZXJjZW50KCkoY291bnRzIC8gc3VtKGNvdW50cykpKSAlPiUKICAjc2VsZWN0KEdlbmRlciwgQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCBwZXJjZW50YWdlKSAlPiUKICBnZ3Bsb3QoYWVzKHg9QVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCB5ID0gcGVyY2VudGFnZSwgZmlsbCA9IEdlbmRlcikpICsKICBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JywgcG9zaXRpb24gPSAnZG9kZ2UnKSAgKyAKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlNldDEiKSArCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHBlcmNlbnRhZ2UpLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0wLjkpLCB2anVzdD0tMC4yNSkgKwogIGdndGl0bGUoIkVmZmVjdCBvZiBhc3Nlc3NtZW50IHR5cGUgb24gZ2VuZGVyIGF3YXJkaW5nIGdhcCIpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0ID0gMC41KSkgKwogIHhsYWIoJ0Fzc2Vzc21lbnQgVHlwZScpICsKICB5bGFiKCdQZXJjZW50YWdlIG9mIHN0dWRlbnRzJykKYGBgCjUlIGRpZmYgaW4gZWFjaCBhc3Nlc3NtZW50IHR5cGUuLi4KRG9lc250IGxvb2sgbGlrZSBleGFtcyBhcmUgbGVhZGluZyB0byBhIHdpZGVyIGdlbmRlciBhd2FyZGluZyBnYXAuIApEaWZmZXJlbmNlIGluIHBlcmZvcm1hbmNlIGZvciBleGFtcyBpcyBzYW1lIGFzIGRpZmZlcmVuY2UgZm9yIGNvdXJzZXdvcmsuLi4uCgpgYGB7cn0KaW50ZXJhY3Rpb24ucGxvdCh4LmZhY3RvciA9IGZ1bGxfZGF0YSRHZW5kZXIsICN4LWF4aXMgdmFyaWFibGUKICAgICAgICAgICAgICAgICB0cmFjZS5mYWN0b3IgPSBmdWxsX2RhdGEkQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDLCAjdmFyaWFibGUgZm9yIGxpbmVzCiAgICAgICAgICAgICAgICAgcmVzcG9uc2UgPSBmdWxsX2RhdGEkTUFSSywgI3ktYXhpcyB2YXJpYWJsZQogICAgICAgICAgICAgICAgIGZ1biA9IG1lZGlhbiwgI21ldHJpYyB0byBwbG90CiAgICAgICAgICAgICAgICAgeWxhYiA9ICJNYXJrIiwKICAgICAgICAgICAgICAgICB4bGFiID0gIkV0aG5pY2l0eSIsCiAgICAgICAgICAgICAgICAgY29sID0gYygiZ3JlZW4iLCAiYmx1ZSIpLAogICAgICAgICAgICAgICAgIGx0eSA9IDEsICNsaW5lIHR5cGUKICAgICAgICAgICAgICAgICBsd2QgPSAyLCAjbGluZSB3aWR0aAogICAgICAgICAgICAgICAgIHRyYWNlLmxhYmVsID0gIkFzc2Vzc21lbnQgdHlwZSIpCmBgYAoKIyBTdGF0aXN0aWNhbCBBbmFseXNpcwoKIyMgTWl4ZWQgTW9kZWwgMwoKYGBge3J9CiMgKipSZXNlYXJjaCBRdWVzdGlvbiAzKioKIyAqKjMuIElzIHRoZXJlIGEgbGluayBiZXR3ZWVuIGFzc2Vzc21lbnQgdHlwZSwgZ2VuZGVyIGFuZCBzdHVkZW50IGFjaGlldmVtZW50PyoqCgptbW9kZWwzPC0gbG1lcihNQVJLIH4gQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDICogR2VuZGVyICsgUFJPR1JBTU1FX1BBUlQgKyAoMXxJRDIpICsgKDF8TU9EVUxFX0NPREUpLCBkYXRhID0gZnVsbF9kYXRhKQpgYGAKCmBgYHtyfQpzdW1tYXJ5KG1tb2RlbDMpCmBgYAoKVGhpcyBpcyBpbnRlcmVzdGluZy4uLnNob3dzIGdhcCBpbiBBVCBzbWFsbGVyIGZvciB3b21hbiBjb21wYXJlZCB0byBtZW4uIApBbHNvIGF2ZXJhZ2UgY3cgZ3JhZGUgdmVyeSBzaW1pbGFyIGZvciBtZW4gYW5kIHdvbWFuCkFuZCB3b21hbiBwZXJmb3JtIGJldHRlciBpbiBleGFtcyBjb21wYXJlZCB0byBtZW4KQW5kIHdvbWFuIHBlcmZvcm0gYmV0dGVyIGluIGV4YW1zIGNvbXBhcmVkIHRvIG1lbi4uLiAKCmBgYHtyfQojaW50ZXJhY3Rpb24gcGxvdCBvZiBtaXhlZCBtb2RlbCAzCnBsb3RfbW9kZWwobW1vZGVsMywgdHlwZSA9ICJwcmVkIiwgdGVybXMgPSBjKCJHZW5kZXIiLCAiQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDIikpCmBgYAoKYGBge3J9CmFub3ZhKG1tb2RlbDMpCmBgYAoKQWdhaW4sIHNpbWlsYXIgdG8gUlExICYgUlEyLCBhbGwgdmFyaWFibGVzIGhvbGQgYSBzaWduaWZpY2FudCBlZmZlY3Qgb24gc3R1ZGVudCBwZXJmb3JtYW5jZQoKYGBge3J9CiNlZmZlY3Qgc2l6ZQplZmZlY3RzaXplOjpldGFfc3F1YXJlZChtbW9kZWwzLCBwYXJ0aWFsID0gVFJVRSkKYGBgCgpgYGB7cn0KI2VtbWVhbnMgaXMgbWVhbnQgdG8gYmUgbG9va2VkIGF0IGZvciBtYWluIGVmZmVjdCBpbiB0aGlzIGNhc2UgdGhleSB3ZXJlIGFsbCBzaWduaWZpY2FudCBzbyB3aWxsIGRvIGZvciBlYWNoPz8KCmVtbWVhbnMobW1vZGVsMywgbGlzdChwYWlyd2lzZX5BU1NFU1NNRU5UX0NPTVBPTkVOVF9UWVBFX0RFU0MpLCBhZGp1c3Q9InR1a2V5IikKYGBgCgpgYGB7cn0KZW1tZWFucyhtbW9kZWwzLCBsaXN0KHBhaXJ3aXNlfkdlbmRlciksIGFkanVzdD0idHVrZXkiKQpgYGAKCmBgYHtyfQplbW1lYW5zKG1tb2RlbDMsIGxpc3QocGFpcndpc2V+UFJPR1JBTU1FX1BBUlQpLCBhZGp1c3Q9InR1a2V5IikKYGBgCgpgYGB7cn0KI1RoZXJlIHdhcyBhbiBpbnRlcmFjdGlvbiBzbyBuZWVkIGVtbWVhbnMgZm9yIHRoaXMKZW1tZWFucyhtbW9kZWwzLCBwYWlyd2lzZSB+IEdlbmRlciB8IEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQykKYGBgCgojIyBFdmFsdWF0aW5nIG1peGVkIG1vZGVsLTMgYXNzdW1wdGlvbnMKCkRvIG5vdCBuZWVkIHRvIGNoZWNrIGxpbmVhcml0eSBhcyBlYWNoIHZhcmlhYmxlIGlzIGNhdGVnb3JpY2FsLgoKIyMjIEhvbW9kc2NlZGFzdGljaXR5CgpgYGB7cn0KI3Bsb3Qgb2YgbWl4ZWQgbW9kZWwgMyByZXNpZHVhbHMKcGxvdChtbW9kZWwzKQpgYGAKCmBgYHtyfQpib3hwbG90KG1tb2RlbDNAcmVzcCR3dHJlc35mdWxsX2RhdGEkQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDKQpgYGAKCmBgYHtyfQpib3hwbG90KG1tb2RlbDNAcmVzcCR3dHJlc35mdWxsX2RhdGEkR2VuZGVyKQpgYGAKCmBgYHtyfQpib3hwbG90KG1tb2RlbDNAcmVzcCR3dHJlc35mdWxsX2RhdGEkUFJPR1JBTU1FX1BBUlQpCmBgYAoKQW5vdGhlciB3YXkgdG8gY2hlY2sgZm9yIGhvbWVvc2NlZGFzdGljaXR5IC0gKHNkKSByZXNpZHVhbHMKCmBgYHtyfQojQVNTRVNTTUVOVCBUWVBFCm1tb2RlbDNyZXM8LW1tb2RlbDNAcmVzcCR3dHJlcwojYXNzZXNzbWVudCB0eXBlIHJlc2lkdWFscyBpbiBkZgptbW9kZWwzZGY8LWRhdGEuZnJhbWUoZnVsbF9kYXRhJEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyxtbW9kZWwzcmVzKSAKbmFtZXMobW1vZGVsM2RmKTwtYygiQVNTRVNTTUVOVF9DT01QT05FTlRfVFlQRV9ERVNDIiwicmVzaWR1YWwiKQptbW9kZWwzZGYgJT4lIGdyb3VwX2J5KEFTU0VTU01FTlRfQ09NUE9ORU5UX1RZUEVfREVTQyklPiUgc3VtbWFyaXNlKHNkKHJlc2lkdWFsKSkKYGBgCgpgYGB7cn0KI0dlbmRlcgptbW9kZWwzZGYyPC1kYXRhLmZyYW1lKGZ1bGxfZGF0YSRHZW5kZXIsbW1vZGVsM3JlcykgCm5hbWVzKG1tb2RlbDNkZjIpPC1jKCJHZW5kZXIiLCJyZXNpZHVhbCIpCm1tb2RlbDNkZjIgJT4lIGdyb3VwX2J5KEdlbmRlciklPiUgc3VtbWFyaXNlKHNkKHJlc2lkdWFsKSkgCmBgYAoKYGBge3J9CiNQcm9ncmFtbWUgcGFydAptbW9kZWwzZGYzPC1kYXRhLmZyYW1lKGZ1bGxfZGF0YSRQUk9HUkFNTUVfUEFSVCxtbW9kZWwzcmVzKSAKbmFtZXMobW1vZGVsM2RmMyk8LWMoIlByb2dyYW1tZV9QYXJ0IiwicmVzaWR1YWwiKQptbW9kZWwzZGYzICU+JSBncm91cF9ieShQcm9ncmFtbWVfUGFydCklPiUgc3VtbWFyaXNlKHNkKHJlc2lkdWFsKSkKYGBgCkFzc3VtcHRpb25zIG1ldCBmb3IgYWxsIHZhcmlhYmxlcwoKIyMjIE5vcm1hbGl0eSBvZiByZXNpZHVhbHMKCmBgYHtyfQpxcW5vcm0obW1vZGVsM3JlcykKYGBgCkFsbCBhc3N1bXB0aW9ucyBmb3IgbWl4ZWQgbW9kZWwgMyBtZXQuIAoKIyMjIENvbmZpZGVuY2UgSW50ZXJ2YWxzCgpgYGB7cn0KI2NvbmZpbnQobW1vZGVsMykKYGBgCgo=